//! Countries and populated places from GeoNames dumps (CC BY 4.0). //! //! Loaded files (see `scripts/fetch-deps.sh`): //! * `countryInfo.txt`, `admin1CodesASCII.txt` //! * `cities*.txt` (worldwide, e.g. cities500) and `full_XX.txt` (every //! populated place of country XX). Duplicates are merged by GeoNames id. use std::collections::HashMap; use std::fs; use std::path::Path; use serde::Serialize; use crate::GeoError; use crate::text::{distance_km, normalize}; #[derive(Debug, Clone, Serialize)] pub struct Country { /// ISO 3166-1 alpha-2 code. pub iso2: String, /// ISO 3166-1 alpha-3 code. pub iso3: String, pub name: String, } #[derive(Debug, Clone, Serialize)] pub struct Place { pub geonames_id: u64, pub name: String, pub country_code: String, /// First-level administrative division (county, state, ...). pub region: Option, pub latitude: f64, pub longitude: f64, pub elevation: Option, pub population: u64, /// GeoNames feature code (PPL, PPLA, PPLC, PPLX, PPLH, ...). pub feature_code: String, /// IANA time zone of the place. pub timezone: String, } #[derive(Debug, Clone, Serialize)] pub struct PlaceMatch { #[serde(flatten)] pub place: Place, /// Which name matched the query. pub matched_name: String, /// Ranking score (higher is better). pub score: f64, } pub struct PlaceIndex { countries: Vec, /// normalized ISO2 / ISO3 / name -> index into `countries` country_keys: HashMap, places: Vec, /// normalized name -> (place index, is primary name) names: HashMap>, by_id: HashMap, by_country: HashMap>, pub downloaded: Option, } fn data_err(what: &Path, reason: impl ToString) -> GeoError { GeoError::Data { what: what.display().to_string(), reason: reason.to_string() } } fn read(path: &Path) -> Result { fs::read_to_string(path).map_err(|e| data_err(path, e)) } impl PlaceIndex { pub fn load(dir: &Path) -> Result { let mut index = PlaceIndex { countries: Vec::new(), country_keys: HashMap::new(), places: Vec::new(), names: HashMap::new(), by_id: HashMap::new(), by_country: HashMap::new(), downloaded: fs::read_to_string(dir.join("DOWNLOADED")).ok().map(|s| s.trim().to_string()), }; index.load_countries(&dir.join("countryInfo.txt"))?; let regions = load_regions(&dir.join("admin1CodesASCII.txt"))?; let mut files: Vec<_> = fs::read_dir(dir) .map_err(|e| data_err(dir, e))? .flatten() .map(|e| e.path()) .filter(|p| { let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default(); name.ends_with(".txt") && (name.starts_with("cities") || name.starts_with("full_")) }) .collect(); // Country files first: they are the most complete for their country. files.sort_by_key(|p| !p.file_name().is_some_and(|n| n.to_string_lossy().starts_with("full_"))); if files.is_empty() { return Err(data_err(dir, "no cities*.txt or full_*.txt GeoNames dump found")); } for file in files { index.load_places(&file, ®ions)?; } Ok(index) } fn load_countries(&mut self, path: &Path) -> Result<(), GeoError> { for line in read(path)?.lines().filter(|l| !l.starts_with('#') && !l.is_empty()) { let f: Vec<&str> = line.split('\t').collect(); if f.len() < 5 { continue; } let idx = self.countries.len(); self.countries.push(Country { iso2: f[0].into(), iso3: f[1].into(), name: f[4].into() }); for key in [f[0], f[1], f[4]] { self.country_keys.entry(normalize(key)).or_insert(idx); } } if self.countries.is_empty() { return Err(data_err(path, "no countries")); } Ok(()) } fn load_places(&mut self, path: &Path, regions: &HashMap) -> Result<(), GeoError> { let content = read(path)?; for line in content.lines() { let f: Vec<&str> = line.split('\t').collect(); // geonameid name asciiname alternatenames lat lon fclass fcode cc cc2 // admin1 admin2 admin3 admin4 population elevation dem timezone moddate if f.len() < 19 || f[6] != "P" { continue; } let Ok(id) = f[0].parse::() else { continue }; if self.by_id.contains_key(&id) { continue; } let (Ok(lat), Ok(lon)) = (f[4].parse::(), f[5].parse::()) else { continue }; let idx = self.places.len() as u32; let country_code = f[8].to_string(); let place = Place { geonames_id: id, name: f[1].to_string(), region: regions.get(&format!("{}.{}", f[8], f[10])).cloned(), latitude: lat, longitude: lon, elevation: f[15].parse().ok().or_else(|| f[16].parse().ok().filter(|&d: &i32| d > -9999)), population: f[14].parse().unwrap_or(0), feature_code: f[7].to_string(), timezone: f[17].to_string(), country_code: country_code.clone(), }; let mut keys: Vec<(String, bool)> = vec![(normalize(f[1]), true), (normalize(f[2]), true)]; keys.extend( f[3].split(',') .filter(|n| n.len() > 1 && !n.contains("http") && !n.chars().all(|c| c.is_ascii_digit())) .map(|n| (normalize(n), false)), ); keys.sort_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(&a.1))); keys.dedup_by(|a, b| a.0 == b.0); for (key, primary) in keys.into_iter().filter(|(k, _)| !k.is_empty()) { self.names.entry(key).or_default().push((idx, primary)); } self.by_id.insert(id, idx); self.by_country.entry(country_code).or_default().push(idx); self.places.push(place); } Ok(()) } pub fn len(&self) -> usize { self.places.len() } pub fn is_empty(&self) -> bool { self.places.is_empty() } pub fn countries(&self) -> &[Country] { &self.countries } /// Looks up a country by ISO alpha-2, alpha-3 or English name. pub fn country(&self, query: &str) -> Result<&Country, GeoError> { self.country_keys .get(&normalize(query)) .map(|&i| &self.countries[i]) .ok_or_else(|| GeoError::UnknownCountry(query.to_string())) } pub fn by_id(&self, id: u64) -> Result<&Place, GeoError> { self.by_id.get(&id).map(|&i| &self.places[i as usize]).ok_or(GeoError::UnknownPlaceId(id)) } /// Searches populated places of a country by name. Exact (normalized) /// matches on any name come first; if there are none, prefix matches are /// returned. `region` optionally filters by first-level division name. pub fn search(&self, country: &Country, query: &str, region: Option<&str>, limit: usize) -> Vec { let q = normalize(query); if q.is_empty() { return Vec::new(); } let region = region.map(normalize).filter(|r| !r.is_empty()); let region_ok = |p: &Place| match (®ion, &p.region) { (None, _) => true, (Some(r), Some(pr)) => normalize(pr).contains(r.as_str()), (Some(_), None) => false, }; let mut matches: Vec = self .names .get(&q) .into_iter() .flatten() .map(|&(i, primary)| (&self.places[i as usize], primary)) .filter(|(p, _)| p.country_code == country.iso2 && region_ok(p)) .map(|(p, primary)| PlaceMatch { matched_name: query.trim().to_string(), score: score(p, if primary { 2.0 } else { 1.0 }), place: p.clone(), }) .collect(); if matches.is_empty() { matches = self .by_country .get(&country.iso2) .into_iter() .flatten() .map(|&i| &self.places[i as usize]) .filter(|p| region_ok(p)) .filter(|p| normalize(&p.name).starts_with(&q)) .map(|p| PlaceMatch { matched_name: p.name.clone(), score: score(p, 0.0), place: p.clone() }) .collect(); } matches.sort_by(|a, b| b.score.total_cmp(&a.score).then(a.place.geonames_id.cmp(&b.place.geonames_id))); matches.dedup_by_key(|m| m.place.geonames_id); matches.truncate(limit); matches } /// Nearest populated place of the country that satisfies `accept`. pub fn nearest( &self, country_iso2: &str, lat: f64, lon: f64, accept: impl Fn(&Place) -> bool, ) -> Option<(&Place, f64)> { self.by_country .get(country_iso2)? .iter() .map(|&i| &self.places[i as usize]) .filter(|p| accept(p)) .map(|p| (p, distance_km(lat, lon, p.latitude, p.longitude))) .min_by(|a, b| a.1.total_cmp(&b.1)) } } /// Ranking: name quality (primary > alternate > prefix), then administrative /// importance, then population. fn score(p: &Place, name_quality: f64) -> f64 { let importance = match p.feature_code.as_str() { "PPLC" => 6.0, "PPLA" => 5.0, "PPLA2" => 4.0, "PPLA3" | "PPLA4" | "PPLA5" | "PPLG" => 3.0, "PPL" => 2.0, "PPLX" | "PPLL" | "PPLS" | "PPLF" => 1.0, // historical/abandoned places last, but still findable (old birth records) "PPLH" | "PPLQ" | "PPLW" => 0.5, _ => 1.0, }; name_quality * 100.0 + importance * 10.0 + (p.population as f64 + 1.0).log10() } fn load_regions(path: &Path) -> Result, GeoError> { Ok(read(path)? .lines() .filter_map(|l| { let mut f = l.split('\t'); Some((f.next()?.to_string(), f.next()?.to_string())) }) .collect()) }