diff --git a/examples/data/location/location_example.vsh b/examples/data/location/location_example.vsh index fd0f85c6..a67c3761 100755 --- a/examples/data/location/location_example.vsh +++ b/examples/data/location/location_example.vsh @@ -1,43 +1,43 @@ -#!/usr/bin/env -S v -n -w -gc none -cc tcc -d use_openssl -enable-globals run +#!/usr/bin/env -S v -n -w -gc none -cg -cc tcc -d use_openssl -enable-globals run import freeflowuniverse.herolib.data.location // Create a new location instance -mut loc := location.new() or { panic(err) } +mut loc := location.new(false) or { panic(err) } println('Location database initialized') // Initialize the database (downloads and imports data) // This only needs to be done once or when updating data println('Downloading and importing location data (this may take a few minutes)...') -loc.init_database() or { panic(err) } +loc.download_and_import() or { panic(err) } println('Data import complete') -// Example 1: Search for a city -println('\nSearching for London...') -results := loc.search('London', 'GB', 5, true) or { panic(err) } -for result in results { - println('${result.city.name}, ${result.country.name} (${result.country.iso2})') - println('Coordinates: ${result.city.latitude}, ${result.city.longitude}') - println('Population: ${result.city.population}') - println('Timezone: ${result.city.timezone}') - println('---') -} +// // Example 1: Search for a city +// println('\nSearching for London...') +// results := loc.search('London', 'GB', 5, true) or { panic(err) } +// for result in results { +// println('${result.city.name}, ${result.country.name} (${result.country.iso2})') +// println('Coordinates: ${result.city.latitude}, ${result.city.longitude}') +// println('Population: ${result.city.population}') +// println('Timezone: ${result.city.timezone}') +// println('---') +// } -// Example 2: Search near coordinates (10km radius from London) -println('\nSearching for cities within 10km of London...') -nearby := loc.search_near(51.5074, -0.1278, 10.0, 5) or { panic(err) } -for result in nearby { - println('${result.city.name}, ${result.country.name}') - println('Distance from center: Approx ${result.similarity:.1f}km') - println('---') -} +// // Example 2: Search near coordinates (10km radius from London) +// println('\nSearching for cities within 10km of London...') +// nearby := loc.search_near(51.5074, -0.1278, 10.0, 5) or { panic(err) } +// for result in nearby { +// println('${result.city.name}, ${result.country.name}') +// println('Distance from center: Approx ${result.similarity:.1f}km') +// println('---') +// } -// Example 3: Fuzzy search in a specific country -println('\nFuzzy searching for "New" in United States...') -us_cities := loc.search('New', 'US', 5, true) or { panic(err) } -for result in us_cities { - println('${result.city.name}, ${result.country.name}') - println('State: ${result.city.admin1_code}') - println('Population: ${result.city.population}') - println('---') -} +// // Example 3: Fuzzy search in a specific country +// println('\nFuzzy searching for "New" in United States...') +// us_cities := loc.search('New', 'US', 5, true) or { panic(err) } +// for result in us_cities { +// println('${result.city.name}, ${result.country.name}') +// println('State: ${result.city.admin1_code}') +// println('Population: ${result.city.population}') +// println('---') +// } diff --git a/lib/core/pathlib/path_tools.v b/lib/core/pathlib/path_tools.v index 072b4f70..7ec836cf 100644 --- a/lib/core/pathlib/path_tools.v +++ b/lib/core/pathlib/path_tools.v @@ -76,7 +76,7 @@ pub fn (mut path Path) expand(dest string) !Path { if path.name().to_lower().ends_with('.tar.gz') || path.name().to_lower().ends_with('.tgz') { cmd := 'tar -xzvf ${path.path} -C ${desto.path}' - console.print_debug(cmd) + //console.print_debug(cmd) res := os.execute(cmd) if res.exit_code > 0 { return error('Could not expand.\n${res}') @@ -136,7 +136,7 @@ pub fn find_common_ancestor(paths_ []string) string { } } paths := paths_.map(os.abs_path(os.real_path(it))) // get the real path (symlinks... resolved) - console.print_debug(paths.str()) + //console.print_debug(paths.str()) parts := paths[0].split('/') mut totest_prev := '/' for i in 1 .. parts.len { @@ -223,7 +223,7 @@ pub fn (mut path Path) move(args MoveArgs) ! { // that last dir needs to move 1 up pub fn (mut path Path) moveup_single_subdir() ! { mut plist := path.list(recursive: false, ignoredefault: true, dirs_only: true)! - console.print_debug(plist.str()) + //console.print_debug(plist.str()) if plist.paths.len != 1 { return error('could not find one subdir in ${path.path} , so cannot move up') } diff --git a/lib/data/location/db.v b/lib/data/location/db.v index be17d19c..c41dc7f3 100644 --- a/lib/data/location/db.v +++ b/lib/data/location/db.v @@ -4,306 +4,45 @@ import db.sqlite import os import encoding.csv import freeflowuniverse.herolib.osal - -const ( - db_file = os.join_path(os.cache_dir(), 'location.db') - geonames_url = 'https://download.geonames.org/export/dump' - cities_url = '${geonames_url}/cities500.zip' -) +import freeflowuniverse.herolib.core.pathlib // LocationDB handles all database operations for locations pub struct LocationDB { mut: db sqlite.DB + tmp_dir pathlib.Path + db_dir pathlib.Path } // new_location_db creates a new LocationDB instance -pub fn new_location_db() !LocationDB { - db := sqlite.connect(db_file)! +pub fn new_location_db(reset bool) !LocationDB { + mut db_dir := pathlib.get_dir(path:'${os.home_dir()}/hero/var/db/location.db',create: true)! + db := sqlite.connect("${db_dir.path}/locations.db")! mut loc_db := LocationDB{ db: db + tmp_dir: pathlib.get_dir(path: '/tmp/location/',create: true)! + db_dir: db_dir } - loc_db.init_tables()! + loc_db.init_tables(reset)! return loc_db } -// init_tables creates the necessary database tables if they don't exist -fn (mut l LocationDB) init_tables() ! { - l.db.exec(' - CREATE TABLE IF NOT EXISTS countries ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - iso2 TEXT NOT NULL, - iso3 TEXT NOT NULL, - continent TEXT, - population INTEGER, - timezone TEXT, - UNIQUE(iso2), - UNIQUE(iso3) - ) - ')! - - l.db.exec(' - CREATE TABLE IF NOT EXISTS cities ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - ascii_name TEXT NOT NULL, - country_id INTEGER NOT NULL, - admin1_code TEXT, - latitude REAL, - longitude REAL, - population INTEGER, - timezone TEXT, - feature_class TEXT, - feature_code TEXT, - search_priority INTEGER DEFAULT 0, - FOREIGN KEY(country_id) REFERENCES countries(id) - ) - ')! - - l.db.exec(' - CREATE TABLE IF NOT EXISTS alternate_names ( - id INTEGER PRIMARY KEY, - city_id INTEGER NOT NULL, - name TEXT NOT NULL, - language_code TEXT, - is_preferred INTEGER, - is_short INTEGER, - FOREIGN KEY(city_id) REFERENCES cities(id) - ) - ')! - - // Create indexes for better search performance - l.db.exec('CREATE INDEX IF NOT EXISTS idx_city_name ON cities(name)')! - l.db.exec('CREATE INDEX IF NOT EXISTS idx_city_ascii ON cities(ascii_name)')! - l.db.exec('CREATE INDEX IF NOT EXISTS idx_city_coords ON cities(latitude, longitude)')! - l.db.exec('CREATE INDEX IF NOT EXISTS idx_alt_name ON alternate_names(name)')! -} - -// download_and_import_data downloads and imports GeoNames data -pub fn (mut l LocationDB) download_and_import_data() ! { - // Download country info - country_file := osal.download( - url: '${geonames_url}/countryInfo.txt' - dest: os.join_path(os.cache_dir(), 'countryInfo.txt') - )! - country_data := os.read_file(country_file.path)! - l.import_country_data(country_data)! - - // Download and process cities - cities_file := osal.download( - url: cities_url - dest: os.join_path(os.cache_dir(), 'cities500.zip') - expand_file: os.join_path(os.cache_dir(), 'cities500.txt') - )! - cities_data := os.read_file(cities_file.path)! - l.import_city_data(cities_data)! -} - -// import_country_data imports country information -fn (mut l LocationDB) import_country_data(data string) ! { - mut tx := l.db.begin()! - - for line in data.split_into_lines() { - if line.starts_with('#') { - continue - } - fields := line.split('\t') - if fields.len < 5 { - continue - } - - tx.exec(' - INSERT OR REPLACE INTO countries ( - iso2, iso3, name, continent, population, timezone - ) VALUES (?, ?, ?, ?, ?, ?) - ', [ - fields[0], // iso2 - fields[1], // iso3 - fields[4], // name - fields[8], // continent - fields[7].i64(), // population - fields[17] // timezone - ])! +// init_tables drops and recreates all tables +fn (mut l LocationDB) init_tables(reset bool) ! { + if reset{ + l.db.exec('DROP TABLE IF EXISTS AlternateName')! + l.db.exec('DROP TABLE IF EXISTS City')! + l.db.exec('DROP TABLE IF EXISTS Country')! } - tx.commit()! -} - -// import_city_data imports city information -fn (mut l LocationDB) import_city_data(data string) ! { - mut tx := l.db.begin()! - - for line in data.split_into_lines() { - fields := line.split('\t') - if fields.len < 15 { - continue - } - - // Get country_id from iso2 code - country_id := l.get_country_id_by_iso2(fields[8]) or { continue } - - tx.exec(' - INSERT OR REPLACE INTO cities ( - id, name, ascii_name, country_id, admin1_code, - latitude, longitude, population, feature_class, - feature_code, timezone - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ', [ - fields[0].int(), // id - fields[1], // name - fields[2], // ascii_name - country_id, - fields[10], // admin1_code - fields[4].f64(), // latitude - fields[5].f64(), // longitude - fields[14].i64(), // population - fields[6], // feature_class - fields[7], // feature_code - fields[17] // timezone - ])! - } - - tx.commit()! -} - -// get_country_id_by_iso2 retrieves a country's ID using its ISO2 code -fn (l LocationDB) get_country_id_by_iso2(iso2 string) !int { - row := l.db.query('SELECT id FROM countries WHERE iso2 = ?', [iso2])! - return row.vals[0].int() -} - -// search_locations searches for locations based on the provided options -pub fn (l LocationDB) search_locations(opts SearchOptions) ![]SearchResult { - mut query := ' - SELECT c.*, co.* - FROM cities c - JOIN countries co ON c.country_id = co.id - WHERE 1=1 - ' - mut params := []string{} - - if opts.query != '' { - if opts.fuzzy { - query += ' AND (c.name LIKE ? OR c.ascii_name LIKE ?)' - params << '%${opts.query}%' - params << '%${opts.query}%' - } else { - query += ' AND (c.name = ? OR c.ascii_name = ?)' - params << opts.query - params << opts.query - } - } - - if opts.country_code != '' { - query += ' AND co.iso2 = ?' - params << opts.country_code - } - - query += ' ORDER BY c.search_priority DESC, c.population DESC LIMIT ?' - params << opts.limit.str() - - rows := l.db.query(query, params)! - mut results := []SearchResult{cap: rows.len} - - for row in rows { - city := City{ - id: row.vals[0].int() - name: row.vals[1] - ascii_name: row.vals[2] - country_id: row.vals[3].int() - admin1_code: row.vals[4] - latitude: row.vals[5].f64() - longitude: row.vals[6].f64() - population: row.vals[7].i64() - timezone: row.vals[8] - feature_class: row.vals[9] - feature_code: row.vals[10] - search_priority: row.vals[11].int() - } - - country := Country{ - id: row.vals[12].int() - name: row.vals[13] - iso2: row.vals[14] - iso3: row.vals[15] - continent: row.vals[16] - population: row.vals[17].i64() - timezone: row.vals[18] - } - - results << SearchResult{ - city: city - country: country - similarity: 1.0 // TODO: implement proper similarity scoring - } - } - - return results -} - -// search_by_coordinates finds locations near the given coordinates -pub fn (l LocationDB) search_by_coordinates(opts CoordinateSearchOptions) ![]SearchResult { - // Use the Haversine formula to calculate distances - query := " - SELECT c.*, co.*, - (6371 * acos(cos(radians(?)) * cos(radians(latitude)) * - cos(radians(longitude) - radians(?)) + sin(radians(?)) * - sin(radians(latitude)))) AS distance - FROM cities c - JOIN countries co ON c.country_id = co.id - HAVING distance < ? - ORDER BY distance - LIMIT ? - " - - rows := l.db.query(query, [ - opts.coordinates.latitude.str(), - opts.coordinates.longitude.str(), - opts.coordinates.latitude.str(), - opts.radius.str(), - opts.limit.str() - ])! - - mut results := []SearchResult{cap: rows.len} - - for row in rows { - city := City{ - id: row.vals[0].int() - name: row.vals[1] - ascii_name: row.vals[2] - country_id: row.vals[3].int() - admin1_code: row.vals[4] - latitude: row.vals[5].f64() - longitude: row.vals[6].f64() - population: row.vals[7].i64() - timezone: row.vals[8] - feature_class: row.vals[9] - feature_code: row.vals[10] - search_priority: row.vals[11].int() - } - - country := Country{ - id: row.vals[12].int() - name: row.vals[13] - iso2: row.vals[14] - iso3: row.vals[15] - continent: row.vals[16] - population: row.vals[17].i64() - timezone: row.vals[18] - } - - results << SearchResult{ - city: city - country: country - similarity: 1.0 - } - } - - return results + sql l.db { + create table Country + create table City + create table AlternateName + }! } // close closes the database connection -pub fn (mut l LocationDB) close() { - l.db.close() +pub fn (mut l LocationDB) close() ! { + l.db.close() or { return err } } diff --git a/lib/data/location/api.v b/lib/data/location/factory.v similarity index 56% rename from lib/data/location/api.v rename to lib/data/location/factory.v index fafbca21..d74473b2 100644 --- a/lib/data/location/api.v +++ b/lib/data/location/factory.v @@ -7,42 +7,18 @@ mut: } // new creates a new Location instance -pub fn new() !Location { - db := new_location_db()! +pub fn new(reset bool) !Location { + db := new_location_db(reset)! return Location{ db: db } } // init_database downloads and imports the initial dataset -pub fn (mut l Location) init_database() ! { +pub fn (mut l Location) download_and_import() ! { l.db.download_and_import_data()! } -// search searches for locations based on the provided options -pub fn (l Location) search(query string, country_code string, limit int, fuzzy bool) ![]SearchResult { - opts := SearchOptions{ - query: query - country_code: country_code - limit: limit - fuzzy: fuzzy - } - return l.db.search_locations(opts) -} - -// search_near searches for locations near the given coordinates -pub fn (l Location) search_near(lat f64, lon f64, radius f64, limit int) ![]SearchResult { - opts := CoordinateSearchOptions{ - coordinates: Coordinates{ - latitude: lat - longitude: lon - } - radius: radius - limit: limit - } - return l.db.search_by_coordinates(opts) -} - // Example usage: /* fn main() ! { diff --git a/lib/data/location/geonames.v b/lib/data/location/geonames.v new file mode 100644 index 00000000..d722e5a0 --- /dev/null +++ b/lib/data/location/geonames.v @@ -0,0 +1,4 @@ +module location + +//https://www.geonames.org/export/codes.html + diff --git a/lib/data/location/importer.v b/lib/data/location/importer.v new file mode 100644 index 00000000..c2b49d72 --- /dev/null +++ b/lib/data/location/importer.v @@ -0,0 +1,253 @@ +module location + +import os +import io +import freeflowuniverse.herolib.osal +import freeflowuniverse.herolib.ui.console +import freeflowuniverse.herolib.core.texttools + +const ( + geonames_url = 'https://download.geonames.org/export/dump' +) + +// download_and_import_data downloads and imports GeoNames data +pub fn (mut l LocationDB) download_and_import_data() ! { + // Download country info + + country_file := osal.download( + url: '${geonames_url}/countryInfo.txt' + dest: '${l.tmp_dir.path}/country.txt' + minsize_kb: 10 + )! + l.import_country_data(country_file.path)! + + l.import_cities()! + +} + +// import_country_data imports country information from a file +fn (mut l LocationDB) import_country_data(filepath string) ! { + console.print_header('Starting import from: ${filepath}') + l.db.exec('BEGIN TRANSACTION')! + + mut file := os.open(filepath) or { + console.print_stderr('Failed to open country file: ${err}') + return err + } + defer { file.close() } + + mut reader := io.new_buffered_reader(reader: file) + defer { reader.free() } + + mut count := 0 + for { + line := reader.read_line() or { break } + if line.starts_with('#') { + continue + } + fields := line.split('\t') + if fields.len < 5 { + continue + } + + iso2 := fields[0] + // Check if country exists + existing_country := sql l.db { + select from Country where iso2 == iso2 + } or { []Country{} } + + country := Country{ + iso2: iso2 + iso3: fields[1] + name: fields[4] + continent: fields[8] + population: fields[7].i64() + timezone: fields[17] + } + + if existing_country.len > 0 { + // Update existing country + sql l.db { + update Country set + iso3 = country.iso3, + name = country.name, + continent = country.continent, + population = country.population, + timezone = country.timezone + where iso2 == iso2 + }! + //console.print_debug("Updated country: ${country}") + } else { + // Insert new country + sql l.db { + insert country into Country + }! + //console.print_debug("Inserted country: ${country}") + } + count++ + if count % 10 == 0 { + console.print_header('Processed ${count} countries') + } + } + + l.db.exec('COMMIT')! + console.print_header('Finished importing countries. Total records: ${count}') +} + +// import_cities imports city information for all countries +fn (mut l LocationDB) import_cities() ! { + console.print_header('Starting Cities Import') + + // Query all countries from the database + + mut countries := sql l.db { + select from Country + }! + + // Process each country + for country in countries { + iso2 := country.iso2.to_upper() + console.print_header('Processing country: ${country.name} (${iso2})') + + // Download and process cities for this country + cities_file := osal.download( + url: '${geonames_url}/${iso2}.zip' + dest: '${l.tmp_dir.path}/${iso2}.zip' + expand_file: '${l.tmp_dir.path}/${iso2}' + minsize_kb: 2 + )! + + println(cities_file) + + l.import_city_data("${l.tmp_dir.path}/${iso2}/${iso2}.txt")! + } +} + +fn (mut l LocationDB) import_city_data(filepath string) ! { + console.print_header('City Import: Starting import from: ${filepath}') + + // the table has the following fields : + // --------------------------------------------------- + // geonameid : integer id of record in geonames database + // name : name of geographical point (utf8) varchar(200) + // asciiname : name of geographical point in plain ascii characters, varchar(200) + // alternatenames : alternatenames, comma separated, ascii names automatically transliterated, convenience attribute from alternatename table, varchar(10000) + // latitude : latitude in decimal degrees (wgs84) + // longitude : longitude in decimal degrees (wgs84) + // feature class : see http://www.geonames.org/export/codes.html, char(1) + // feature code : see http://www.geonames.org/export/codes.html, varchar(10) + // country code : ISO-3166 2-letter country code, 2 characters + // cc2 : alternate country codes, comma separated, ISO-3166 2-letter country code, 200 characters + // admin1 code : fipscode (subject to change to iso code), see exceptions below, see file admin1Codes.txt for display names of this code; varchar(20) + // admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80) + // admin3 code : code for third level administrative division, varchar(20) + // admin4 code : code for fourth level administrative division, varchar(20) + // population : bigint (8 byte int) + // elevation : in meters, integer + // dem : digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat. + // timezone : the iana timezone id (see file timeZone.txt) varchar(40) + // modification date : date of last modification in yyyy-MM-dd format + + + + l.db.exec('BEGIN TRANSACTION')! + + mut file := os.open(filepath) or { + console.print_stderr('Failed to open city file: ${err}') + return err + } + defer { file.close() } + + mut reader := io.new_buffered_reader(reader:file) + defer { reader.free() } + + mut count := 0 + console.print_header('Start import ${filepath}') + for { + line := reader.read_line() or { + //console.print_debug('End of file reached') + break + } + //console.print_debug(line) + fields := line.split('\t') + if fields.len < 12 { // Need at least 12 fields for required data + console.print_stderr('fields < 12: ${line}') + continue + } + + // Parse fields according to geonames format + geoname_id := fields[0].int() + name := fields[1] + ascii_name := texttools.name_fix(fields[2]) + country_iso2 := fields[8].to_upper() + + // Check if city exists + existing_city := sql l.db { + select from City where id == geoname_id + } or { []City{} } + + city := City{ + id: geoname_id + name: name + ascii_name: ascii_name + country_iso2: country_iso2 + postal_code: '' // Not provided in this format + state_name: '' // Will need separate admin codes file + state_code: fields[10] + county_name: '' + county_code: fields[11] + community_name: '' + community_code: '' + latitude: fields[4].f64() + longitude: fields[5].f64() + accuracy: 4 // Using geonameid, so accuracy is 4 + population: fields[14].i64() + timezone: fields[17] + feature_class: fields[6] + feature_code: fields[7] + search_priority: 0 // Default priority + } + + if existing_city.len > 0 { + // Update existing city + sql l.db { + update City set + name = city.name, + ascii_name = city.ascii_name, + country_iso2 = city.country_iso2, + postal_code = city.postal_code, + state_name = city.state_name, + state_code = city.state_code, + county_name = city.county_name, + county_code = city.county_code, + community_name = city.community_name, + community_code = city.community_code, + latitude = city.latitude, + longitude = city.longitude, + accuracy = city.accuracy, + population = city.population, + timezone = city.timezone, + feature_class = city.feature_class, + feature_code = city.feature_code, + search_priority = city.search_priority + where id == geoname_id + }! + //console.print_debug("Updated city: ${city}") + } else { + // Insert new city + sql l.db { + insert city into City + }! + //console.print_debug("Inserted city: ${city}") + } + count++ + // if count % 1000 == 0 { + // console.print_header( 'Processed ${count} cities') + // } + } + + console.print_debug( 'Processed ${count} cities') + + l.db.exec('COMMIT')! + console.print_header('Finished importing cities for ${filepath}. Total records: ${count}') +} diff --git a/lib/data/location/models.v b/lib/data/location/models.v index 3bc7659b..1f8e305d 100644 --- a/lib/data/location/models.v +++ b/lib/data/location/models.v @@ -2,37 +2,43 @@ module location pub struct Country { pub: - id int [primary] - name string [required] - iso2 string [required; sql: 'iso2'; max_len: 2] - iso3 string [required; sql: 'iso3'; max_len: 3] - continent string [max_len: 2] + iso2 string @[primary; sql: 'iso2'; max_len: 2; unique; index] + name string @[required; unique; index] + iso3 string @[required; sql: 'iso3'; max_len: 3; unique; index] + continent string @[max_len: 2] population i64 - timezone string [max_len: 40] + timezone string @[max_len: 40] } pub struct City { pub: - id int [primary] - name string [required; max_len: 200] - ascii_name string [required; max_len: 200] // Normalized name without special characters - country_id int [required] - admin1_code string [max_len: 20] // State/Province code - latitude f64 - longitude f64 + id int @[unique; index] + name string @[required; max_len: 200; index] + ascii_name string @[required; max_len: 200; index] // Normalized name without special characters + country_iso2 string @[required; fkey: 'Country.iso2'] + postal_code string @[max_len: 20; index ] //postal code + state_name string @[max_len: 100] // State/Province name + state_code string @[max_len: 20] // State/Province code + county_name string @[max_len: 100] + county_code string @[max_len: 20] + community_name string @[max_len: 100] + community_code string @[max_len: 20] + latitude f64 @[index: 'idx_coords'] + longitude f64 @[index: 'idx_coords'] population i64 - timezone string [max_len: 40] - feature_class string [max_len: 1] // For filtering (P for populated places) - feature_code string [max_len: 10] // Detailed type (PPL, PPLA, etc.) + timezone string @[max_len: 40] + feature_class string @[max_len: 1] // For filtering (P for populated places) + feature_code string @[max_len: 10] // Detailed type (PPL, PPLA, etc.) search_priority int + accuracy u8 = 1 //1=estimated, 4=geonameid, 6=centroid of addresses or shape } pub struct AlternateName { pub: - id int [primary] - city_id int [required] - name string [required; max_len: 200] - language_code string [max_len: 2] + id int @[primary; sql: serial] + city_id int @[required; fkey: 'City.id'] + name string @[required; max_len: 200; index] + language_code string @[max_len: 2] is_preferred bool is_short bool } diff --git a/lib/data/location/search.v b/lib/data/location/search.v new file mode 100644 index 00000000..5ba3d32e --- /dev/null +++ b/lib/data/location/search.v @@ -0,0 +1,169 @@ +module location + +import db.sqlite + + +// search searches for locations based on the provided options +// pub fn (l Location) search(query string, country_code string, limit int, fuzzy bool) ![]SearchResult { +// opts := SearchOptions{ +// query: query +// country_code: country_code +// limit: limit +// fuzzy: fuzzy +// } +// return l.db.search_locations(opts) +// } + +// // search_near searches for locations near the given coordinates +// pub fn (l Location) search_near(lat f64, lon f64, radius f64, limit int) ![]SearchResult { +// opts := CoordinateSearchOptions{ +// coordinates: Coordinates{ +// latitude: lat +// longitude: lon +// } +// radius: radius +// limit: limit +// } +// return l.db.search_by_coordinates(opts) +// } + + +// // search_locations searches for locations based on the provided options +// pub fn (l LocationDB) search_locations(opts SearchOptions) ![]SearchResult { +// mut query_conditions := []string{} +// mut params := []string{} + +// if opts.query != '' { +// if opts.fuzzy { +// query_conditions << '(c.name LIKE ? OR c.ascii_name LIKE ?)' +// params << '%${opts.query}%' +// params << '%${opts.query}%' +// } else { +// query_conditions << '(c.name = ? OR c.ascii_name = ?)' +// params << opts.query +// params << opts.query +// } +// } + +// if opts.country_code != '' { +// query_conditions << 'co.iso2 = ?' +// params << opts.country_code +// } + +// where_clause := if query_conditions.len > 0 { 'WHERE ' + query_conditions.join(' AND ') } else { '' } + +// query := ' +// SELECT c.*, co.* +// FROM City c +// JOIN Country co ON c.country_id = co.id +// ${where_clause} +// ORDER BY c.search_priority DESC, c.population DESC +// LIMIT ${opts.limit} +// ' + +// query_with_params := sql l.db { +// raw(query, params) +// }! +// rows := l.db.exec(query_with_params)! +// mut results := []SearchResult{cap: rows.len} + +// for row in rows { +// city := City{ +// id: row.vals[0].int() +// name: row.vals[1] +// ascii_name: row.vals[2] +// country_id: row.vals[3].int() +// admin1_code: row.vals[4] +// latitude: row.vals[5].f64() +// longitude: row.vals[6].f64() +// population: row.vals[7].i64() +// timezone: row.vals[8] +// feature_class: row.vals[9] +// feature_code: row.vals[10] +// search_priority: row.vals[11].int() +// } + +// country := Country{ +// id: row.vals[12].int() +// name: row.vals[13] +// iso2: row.vals[14] +// iso3: row.vals[15] +// continent: row.vals[16] +// population: row.vals[17].i64() +// timezone: row.vals[18] +// } + +// results << SearchResult{ +// city: city +// country: country +// similarity: 1.0 // TODO: implement proper similarity scoring +// } +// } + +// return results +// } + +// // search_by_coordinates finds locations near the given coordinates +// pub fn (l LocationDB) search_by_coordinates(opts CoordinateSearchOptions) ![]SearchResult { +// // Use the Haversine formula to calculate distances +// query := " +// SELECT c.*, co.*, +// (6371 * acos(cos(radians(?)) * cos(radians(latitude)) * +// cos(radians(longitude) - radians(?)) + sin(radians(?)) * +// sin(radians(latitude)))) AS distance +// FROM City c +// JOIN Country co ON c.country_id = co.id +// HAVING distance < ? +// ORDER BY distance +// LIMIT ? +// " + +// params := [ +// opts.coordinates.latitude.str(), +// opts.coordinates.longitude.str(), +// opts.coordinates.latitude.str(), +// opts.radius.str(), +// opts.limit.str() +// ] +// query_with_params := sql l.db { +// raw(query, params) +// }! +// rows := l.db.exec(query_with_params)! + +// mut results := []SearchResult{cap: rows.len} + +// for row in rows { +// city := City{ +// id: row.vals[0].int() +// name: row.vals[1] +// ascii_name: row.vals[2] +// country_id: row.vals[3].int() +// admin1_code: row.vals[4] +// latitude: row.vals[5].f64() +// longitude: row.vals[6].f64() +// population: row.vals[7].i64() +// timezone: row.vals[8] +// feature_class: row.vals[9] +// feature_code: row.vals[10] +// search_priority: row.vals[11].int() +// } + +// country := Country{ +// id: row.vals[12].int() +// name: row.vals[13] +// iso2: row.vals[14] +// iso3: row.vals[15] +// continent: row.vals[16] +// population: row.vals[17].i64() +// timezone: row.vals[18] +// } + +// results << SearchResult{ +// city: city +// country: country +// similarity: 1.0 +// } +// } + +// return results +// } diff --git a/lib/osal/downloader.v b/lib/osal/downloader.v index cfe4e0cd..fabe6bf2 100644 --- a/lib/osal/downloader.v +++ b/lib/osal/downloader.v @@ -26,6 +26,11 @@ pub mut: pub fn download(args_ DownloadArgs) !pathlib.Path { mut args := args_ + args.dest = args.dest.trim(" ").trim_right("/") + args.expand_dir = args.expand_dir.trim(" ").trim_right("/") + args.expand_file = args.expand_file.replace("//","/") + args.dest = args.dest.replace("//","/") + console.print_header('download: ${args.url}') if args.name == '' { if args.dest != '' { @@ -38,7 +43,7 @@ pub fn download(args_ DownloadArgs) !pathlib.Path { args.name = lastname } if args.name == '' { - return error('cannot find name for download') + return error('cannot find name for download of \n\'${args_}\'') } } @@ -69,8 +74,30 @@ pub fn download(args_ DownloadArgs) !pathlib.Path { } if args.reset { - mut dest_delete := pathlib.get_file(path: args.dest + '_', check: false)! - dest_delete.delete()! + // Clean up all related files when resetting + if os.exists(args.dest) { + if os.is_dir(args.dest) { + os.rmdir_all(args.dest) or { } + } else { + os.rm(args.dest) or { } + } + } + if os.exists(args.dest + '_') { + if os.is_dir(args.dest + '_') { + os.rmdir_all(args.dest + '_') or { } + } else { + os.rm(args.dest + '_') or { } + } + } + if os.exists(args.dest + '.meta') { + if os.is_dir(args.dest + '.meta') { + os.rmdir_all(args.dest + '.meta') or { } + } else { + os.rm(args.dest + '.meta') or { } + } + } + // Recreate meta file after cleanup + meta = pathlib.get_file(path: args.dest + '.meta', create: true)! } meta.write(args.url.trim_space())! @@ -89,8 +116,15 @@ pub fn download(args_ DownloadArgs) !pathlib.Path { if todownload { mut dest0 := pathlib.get_file(path: args.dest + '_')! + // Clean up any existing temporary file/directory before download + if os.exists(dest0.path) { + if os.is_dir(dest0.path) { + os.rmdir_all(dest0.path) or { } + } else { + os.rm(dest0.path) or { } + } + } cmd := ' - rm -f ${dest0.path} cd /tmp curl -L \'${args.url}\' -o ${dest0.path} ' @@ -121,15 +155,26 @@ pub fn download(args_ DownloadArgs) !pathlib.Path { dest.check() } if args.expand_dir.len > 0 { + // Clean up directory if it exists if os.exists(args.expand_dir) { - os.rmdir_all(args.expand_dir)! + os.rmdir_all(args.expand_dir) or { + return error('Failed to remove existing directory ${args.expand_dir}: ${err}') + } } - return dest.expand(args.expand_dir)! } if args.expand_file.len > 0 { + // Clean up file/directory if it exists if os.exists(args.expand_file) { - os.rm(args.expand_file)! + if os.is_dir(args.expand_file) { + os.rmdir_all(args.expand_file) or { + return error('Failed to remove existing directory ${args.expand_file}: ${err}') + } + } else { + os.rm(args.expand_file) or { + return error('Failed to remove existing file ${args.expand_file}: ${err}') + } + } } return dest.expand(args.expand_file)! } diff --git a/test_sqlite.v b/test_sqlite.v new file mode 100644 index 00000000..9a1a89da --- /dev/null +++ b/test_sqlite.v @@ -0,0 +1,7 @@ +import db.sqlite + +fn main() { + db := sqlite.connect(':memory:')! + println('SQLite connection successful') + db.close()! +}