MahmoudEmad/herolib
1
0
Fork 0
Code Issues 55 Pull Requests 1 Actions Packages Projects Releases 26 Wiki Activity

location is importing in psql

Browse Source
This commit is contained in:
despiegk
2025-02-06 06:48:47 +03:00
parent 048c72b294
commit 6bdc4a5b8f
8 changed files with 220 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/postgresql/docker-compose.yml
View File

@@ -3,6 +3,8 @@ services:
db: db:
image: 'postgres:17.2-alpine3.21' image: 'postgres:17.2-alpine3.21'
restart: always restart: always
ports:
- 5432:5432
environment: environment:
POSTGRES_PASSWORD: 1234 POSTGRES_PASSWORD: 1234
networks: networks:

5
docker/postgresql/start.sh Normal file → Executable file
View File

@@ -4,5 +4,10 @@
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$SCRIPT_DIR" cd "$SCRIPT_DIR"
# Stop any existing containers and remove them
docker compose down
# Start the services in detached mode
docker compose up -d
echo "PostgreSQL is ready"

BIN
examples/data/location/location_example Executable file
View File

Binary file not shown.

2
examples/virt/docker/ai_web_ui/readme.md
View File

@@ -2,7 +2,7 @@
- make docker build (see docker_ubuntu example) - make docker build (see docker_ubuntu example)
- start from docker_ubuntu - start from docker_ubuntu
- for build use our vlanf approach (example see docker_ubuntu, make sure we have our zinit & ssh working) - for build use our vlang approach (example see docker_ubuntu, make sure we have our zinit & ssh working)
- install the web UI: openwebui (not by docker but use uv to install this software) - install the web UI: openwebui (not by docker but use uv to install this software)
- use https://github.com/astral-sh/uv for the python part - use https://github.com/astral-sh/uv for the python part
- as last step, clean it all up (remove apt cache, ...) - as last step, clean it all up (remove apt cache, ...)

57
lib/data/location/db.v
View File

@@ -1,6 +1,6 @@
module location module location
import db.sqlite import db.pg
import os import os
import encoding.csv import encoding.csv
import freeflowuniverse.herolib.osal import freeflowuniverse.herolib.osal
@@ -9,7 +9,7 @@ import freeflowuniverse.herolib.core.pathlib
// LocationDB handles all database operations for locations // LocationDB handles all database operations for locations
pub struct LocationDB { pub struct LocationDB {
mut: mut:
db sqlite.DB db pg.DB
tmp_dir pathlib.Path tmp_dir pathlib.Path
db_dir pathlib.Path db_dir pathlib.Path
} }
@@ -17,7 +17,48 @@ mut:
// new_location_db creates a new LocationDB instance // new_location_db creates a new LocationDB instance
pub fn new_location_db(reset bool) !LocationDB { pub fn new_location_db(reset bool) !LocationDB {
mut db_dir := pathlib.get_dir(path:'${os.home_dir()}/hero/var/db/location.db',create: true)! mut db_dir := pathlib.get_dir(path:'${os.home_dir()}/hero/var/db/location.db',create: true)!
db := sqlite.connect("${db_dir.path}/locations.db")!
// PostgreSQL connection parameters with defaults
mut host := os.getenv('POSTGRES_HOST')
if host == '' {
host = 'localhost'
}
port := os.getenv('POSTGRES_PORT')
port_num := if port == '' { 5432 } else { port.int() }
mut user := os.getenv('POSTGRES_USER')
if user == '' {
user = 'postgres'
}
mut password := os.getenv('POSTGRES_PASSWORD')
if password == '' {
password = '1234'
}
mut dbname := os.getenv('POSTGRES_DB')
if dbname == '' {
dbname = 'locations'
}
// First try to connect to create the database if it doesn't exist
mut init_db := pg.connect(
host: host
port: port_num
user: user
password: password
dbname: 'postgres'
) or { return error('Failed to connect to PostgreSQL: ${err}') }
init_db.exec("CREATE DATABASE ${dbname}") or {}
init_db.close()
// Now connect to our database
db := pg.connect(
host: host
port: port_num
user: user
password: password
dbname: dbname
) or { return error('Failed to connect to PostgreSQL: ${err}') }
mut loc_db := LocationDB{ mut loc_db := LocationDB{
db: db db: db
tmp_dir: pathlib.get_dir(path: '/tmp/location/',create: true)! tmp_dir: pathlib.get_dir(path: '/tmp/location/',create: true)!
@@ -30,9 +71,11 @@ pub fn new_location_db(reset bool) !LocationDB {
// init_tables drops and recreates all tables // init_tables drops and recreates all tables
fn (mut l LocationDB) init_tables(reset bool) ! { fn (mut l LocationDB) init_tables(reset bool) ! {
if reset { if reset {
l.db.exec('DROP TABLE IF EXISTS AlternateName')! sql l.db {
l.db.exec('DROP TABLE IF EXISTS City')! drop table AlternateName
l.db.exec('DROP TABLE IF EXISTS Country')! drop table City
drop table Country
}!
} }
sql l.db { sql l.db {
@@ -49,5 +92,5 @@ fn (mut l LocationDB) init_tables(reset bool) ! {
// close closes the database connection // close closes the database connection
pub fn (mut l LocationDB) close() ! { pub fn (mut l LocationDB) close() ! {
l.db.close() or { return err } l.db.close()
} }

2
lib/data/location/models.v
View File

@@ -31,7 +31,7 @@ pub:
feature_class string @[max_len: 1] // For filtering (P for populated places) feature_class string @[max_len: 1] // For filtering (P for populated places)
feature_code string @[max_len: 10] // Detailed type (PPL, PPLA, etc.) feature_code string @[max_len: 10] // Detailed type (PPL, PPLA, etc.)
search_priority int search_priority int
accuracy u8 = 1 //1=estimated, 4=geonameid, 6=centroid of addresses or shape accuracy i16 = 1 //1=estimated, 4=geonameid, 6=centroid of addresses or shape
} }
pub struct AlternateName { pub struct AlternateName {

5
lib/data/location/readme.md Normal file
View File

@@ -0,0 +1,5 @@
make sure to do
brew install libpq

312
lib/data/location/search.v
View File

@@ -1,177 +1,177 @@
module location module location
import db.sqlite import db.pg
// // search searches for locations based on the provided options
// pub fn (l Location) search(query string, country_code string, limit int, fuzzy bool) ![]SearchResult {
// opts := SearchOptions{
// query: query
// country_code: country_code
// limit: limit
// fuzzy: fuzzy
// }
// return l.db.search_locations(opts)
// }
// search searches for locations based on the provided options // // search_near searches for locations near the given coordinates
pub fn (l Location) search(query string, country_code string, limit int, fuzzy bool) ![]SearchResult { // pub fn (l Location) search_near(lat f64, lon f64, radius f64, limit int) ![]SearchResult {
opts := SearchOptions{ // opts := CoordinateSearchOptions{
query: query // coordinates: Coordinates{
country_code: country_code // latitude: lat
limit: limit // longitude: lon
fuzzy: fuzzy // }
} // radius: radius
return l.db.search_locations(opts) // limit: limit
} // }
// return l.db.search_by_coordinates(opts)
// }
// search_near searches for locations near the given coordinates // // search_locations searches for locations based on the provided options
pub fn (l Location) search_near(lat f64, lon f64, radius f64, limit int) ![]SearchResult { // pub fn (l LocationDB) search_locations(opts SearchOptions) ![]SearchResult {
opts := CoordinateSearchOptions{ // mut query_conditions := []string{}
coordinates: Coordinates{ // mut params := []string{}
latitude: lat
longitude: lon
}
radius: radius
limit: limit
}
return l.db.search_by_coordinates(opts)
}
// if opts.query != '' {
// if opts.fuzzy {
// query_conditions << '(c.name ILIKE $${params.len + 1} OR c.ascii_name ILIKE $${params.len + 2})'
// params << '%${opts.query}%'
// params << '%${opts.query}%'
// } else {
// query_conditions << '(c.name = $${params.len + 1} OR c.ascii_name = $${params.len + 2})'
// params << opts.query
// params << opts.query
// }
// }
// search_locations searches for locations based on the provided options // if opts.country_code != '' {
pub fn (l LocationDB) search_locations(opts SearchOptions) ![]SearchResult { // query_conditions << 'c.country_iso2 = $${params.len + 1}'
mut query_conditions := []string{} // params << opts.country_code
mut params := []string{} // }
if opts.query != '' { // where_clause := if query_conditions.len > 0 { 'WHERE ' + query_conditions.join(' AND ') } else { '' }
if opts.fuzzy {
query_conditions << '(c.name LIKE ? OR c.ascii_name LIKE ?)'
params << '%${opts.query}%'
params << '%${opts.query}%'
} else {
query_conditions << '(c.name = ? OR c.ascii_name = ?)'
params << opts.query
params << opts.query
}
}
if opts.country_code != '' { // query := '
query_conditions << 'c.country_iso2 = ?' // SELECT c.*, co.*
params << opts.country_code // FROM City c
} // JOIN Country co ON c.country_iso2 = co.iso2
// ${where_clause}
// ORDER BY c.search_priority DESC, c.population DESC
// LIMIT ${opts.limit}
// '
where_clause := if query_conditions.len > 0 { 'WHERE ' + query_conditions.join(' AND ') } else { '' } // rows := l.db.exec_param_many(query, params)!
// mut results := []SearchResult{cap: rows.len}
query := ' // for row in rows {
SELECT c.*, co.* // city := City{
FROM City c // id: row.vals[0].int() or { 0 }
JOIN Country co ON c.country_iso2 = co.iso2 // name: row.vals[1] or { '' }
${where_clause} // ascii_name: row.vals[2] or { '' }
ORDER BY c.search_priority DESC, c.population DESC // country_iso2: row.vals[3] or { '' }
LIMIT ${opts.limit} // postal_code: row.vals[4] or { '' }
' // state_name: row.vals[5] or { '' }
// state_code: row.vals[6] or { '' }
// county_name: row.vals[7] or { '' }
// county_code: row.vals[8] or { '' }
// community_name: row.vals[9] or { '' }
// community_code: row.vals[10] or { '' }
// latitude: row.vals[11].f64() or { 0.0 }
// longitude: row.vals[12].f64() or { 0.0 }
// population: row.vals[13].i64() or { 0 }
// timezone: row.vals[14] or { '' }
// feature_class: row.vals[15] or { '' }
// feature_code: row.vals[16] or { '' }
// search_priority: row.vals[17].int() or { 0 }
// accuracy: u8(row.vals[18].int() or { 1 })
// }
rows := l.db.exec_param_many(query, params)! // country := Country{
mut results := []SearchResult{cap: rows.len} // iso2: row.vals[19] or { '' }
// name: row.vals[20] or { '' }
// iso3: row.vals[21] or { '' }
// continent: row.vals[22] or { '' }
// population: row.vals[23].i64() or { 0 }
// timezone: row.vals[24] or { '' }
// import_date: row.vals[25].i64() or { 0 }
// }
for row in rows { // results << SearchResult{
city := City{ // city: city
id: row.vals[0].int() // country: country
name: row.vals[1] // similarity: 1.0 // TODO: implement proper similarity scoring
ascii_name: row.vals[2] // }
country_iso2: row.vals[3] // }
postal_code: row.vals[4]
state_name: row.vals[5]
state_code: row.vals[6]
county_name: row.vals[7]
county_code: row.vals[8]
community_name: row.vals[9]
community_code: row.vals[10]
latitude: row.vals[11].f64()
longitude: row.vals[12].f64()
population: row.vals[13].i64()
timezone: row.vals[14]
feature_class: row.vals[15]
feature_code: row.vals[16]
search_priority: row.vals[17].int()
accuracy: row.vals[18].u8()
}
country := Country{ // return results
iso2: row.vals[19] // }
name: row.vals[20]
iso3: row.vals[21]
continent: row.vals[22]
population: row.vals[23].i64()
timezone: row.vals[24]
import_date: row.vals[25].i64()
}
results << SearchResult{ // // search_by_coordinates finds locations near the given coordinates
city: city // pub fn (l LocationDB) search_by_coordinates(opts CoordinateSearchOptions) ![]SearchResult {
country: country // // Use the Haversine formula to calculate distances
similarity: 1.0 // TODO: implement proper similarity scoring // query := "
} // WITH distances AS (
} // SELECT c.*, co.*,
// (6371 * acos(cos(radians($1)) * cos(radians(latitude)) *
return results // cos(radians(longitude) - radians($2)) + sin(radians($1)) *
} // sin(radians(latitude)))) AS distance
// FROM City c
// search_by_coordinates finds locations near the given coordinates // JOIN Country co ON c.country_iso2 = co.iso2
pub fn (l LocationDB) search_by_coordinates(opts CoordinateSearchOptions) ![]SearchResult { // )
// Use the Haversine formula to calculate distances // SELECT * FROM distances
query := " // WHERE distance < $3
SELECT c.*, co.*, // ORDER BY distance
(6371 * acos(cos(radians(?)) * cos(radians(latitude)) * // LIMIT $4
cos(radians(longitude) - radians(?)) + sin(radians(?)) * // "
sin(radians(latitude)))) AS distance
FROM City c
JOIN Country co ON c.country_iso2 = co.iso2
HAVING distance < ?
ORDER BY distance
LIMIT ?
"
params := [ // params := [
opts.coordinates.latitude.str(), // opts.coordinates.latitude.str(),
opts.coordinates.longitude.str(), // opts.coordinates.longitude.str(),
opts.coordinates.latitude.str(), // opts.radius.str(),
opts.radius.str(), // opts.limit.str()
opts.limit.str() // ]
] // rows := l.db.exec_param_many(query, params)!
rows := l.db.exec_param_many(query, params)!
mut results := []SearchResult{cap: rows.len} // mut results := []SearchResult{cap: rows.len}
for row in rows { // for row in rows {
city := City{ // city := City{
id: row.vals[0].int() // id: row.vals[0].int() or { 0 }
name: row.vals[1] // name: row.vals[1] or { '' }
ascii_name: row.vals[2] // ascii_name: row.vals[2] or { '' }
country_iso2: row.vals[3] // country_iso2: row.vals[3] or { '' }
postal_code: row.vals[4] // postal_code: row.vals[4] or { '' }
state_name: row.vals[5] // state_name: row.vals[5] or { '' }
state_code: row.vals[6] // state_code: row.vals[6] or { '' }
county_name: row.vals[7] // county_name: row.vals[7] or { '' }
county_code: row.vals[8] // county_code: row.vals[8] or { '' }
community_name: row.vals[9] // community_name: row.vals[9] or { '' }
community_code: row.vals[10] // community_code: row.vals[10] or { '' }
latitude: row.vals[11].f64() // latitude: row.vals[11].f64() or { 0.0 }
longitude: row.vals[12].f64() // longitude: row.vals[12].f64() or { 0.0 }
population: row.vals[13].i64() // population: row.vals[13].i64() or { 0 }
timezone: row.vals[14] // timezone: row.vals[14] or { '' }
feature_class: row.vals[15] // feature_class: row.vals[15] or { '' }
feature_code: row.vals[16] // feature_code: row.vals[16] or { '' }
search_priority: row.vals[17].int() // search_priority: row.vals[17].int() or { 0 }
accuracy: row.vals[18].u8() // accuracy: u8(row.vals[18].int() or { 1 })
} // }
country := Country{ // country := Country{
iso2: row.vals[19] // iso2: row.vals[19] or { '' }
name: row.vals[20] // name: row.vals[20] or { '' }
iso3: row.vals[21] // iso3: row.vals[21] or { '' }
continent: row.vals[22] // continent: row.vals[22] or { '' }
population: row.vals[23].i64() // population: row.vals[23].i64() or { 0 }
timezone: row.vals[24] // timezone: row.vals[24] or { '' }
import_date: row.vals[25].i64() // import_date: row.vals[25].i64() or { 0 }
} // }
results << SearchResult{ // results << SearchResult{
city: city // city: city
country: country // country: country
similarity: 1.0 // similarity: 1.0
} // }
} // }
return results // return results
} // }