MatthiasQuintern/regina
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added region table to db

Browse Source
This commit is contained in:
matthias@arch 2023-05-18 03:18:49 +02:00
parent b604a92b59
commit d6a033aa5b
3 changed files with 190 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
database.uxf
View File

@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<diagram program="umlet" version="15.1">
<zoom_level>9</zoom_level>
<zoom_level>12</zoom_level>
<element>
<id>UMLClass</id>
<coordinates>
<x>441</x>
<y>306</y>
<w>180</w>
<h>162</h>
<x>396</x>
<y>240</y>
<w>240</w>
<h>216</h>
</coordinates>
<panel_attributes>visitor
--
@ -27,10 +27,10 @@ bg=MAGENTA</panel_attributes>
<element>
<id>Relation</id>
<coordinates>
<x>558</x>
<y>216</y>
<w>36</w>
<h>108</h>
<x>552</x>
<y>120</y>
<w>48</w>
<h>144</h>
</coordinates>
<panel_attributes>lt=-
m1=n
@ -41,10 +41,10 @@ m2=1
<element>
<id>UMLClass</id>
<coordinates>
<x>702</x>
<y>306</y>
<w>171</w>
<h>153</h>
<x>744</x>
<y>240</y>
<w>228</w>
<h>204</h>
</coordinates>
<panel_attributes>request
--
@ -64,10 +64,10 @@ bg=CYAN</panel_attributes>
<element>
<id>Relation</id>
<coordinates>
<x>612</x>
<y>333</y>
<w>108</w>
<h>45</h>
<x>624</x>
<y>276</y>
<w>144</w>
<h>60</h>
</coordinates>
<panel_attributes>lt=-
m1=1
@ -78,10 +78,10 @@ m2=n
<element>
<id>UMLClass</id>
<coordinates>
<x>333</x>
<y>135</y>
<w>180</w>
<h>90</h>
<x>252</x>
<y>12</y>
<w>240</w>
<h>120</h>
</coordinates>
<panel_attributes>platform
--
@ -97,38 +97,19 @@ bg=MAGENTA</panel_attributes>
<element>
<id>UMLClass</id>
<coordinates>
<x>702</x>
<y>549</y>
<w>171</w>
<h>117</h>
<x>744</x>
<y>564</y>
<w>228</w>
<h>156</h>
</coordinates>
<panel_attributes>city
--
&lt;&lt;PK&gt;&gt;
- city_id: INTEGER
--
- country_id: INTEGER
- name: TEXT
- region: TEXT
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
</element>
<element>
<id>UMLClass</id>
<coordinates>
<x>945</x>
<y>549</y>
<w>171</w>
<h>99</h>
</coordinates>
<panel_attributes>country
--
&lt;&lt;PK&gt;&gt;
- country_id: INTEGER
--
- name: TEXT UNIQUE
- code: TEXT UNIQUE
- region_id: INTEGER
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
@ -136,10 +117,10 @@ bg=ORANGE</panel_attributes>
<element>
<id>Relation</id>
<coordinates>
<x>864</x>
<y>567</y>
<w>99</w>
<h>45</h>
<x>960</x>
<y>588</y>
<w>132</w>
<h>60</h>
</coordinates>
<panel_attributes>lt=-
m1=1
@ -150,10 +131,10 @@ m2=n
<element>
<id>Relation</id>
<coordinates>
<x>612</x>
<y>567</y>
<w>108</w>
<h>45</h>
<x>624</x>
<y>588</y>
<w>144</w>
<h>60</h>
</coordinates>
<panel_attributes>lt=-
m1=1
@ -164,10 +145,10 @@ m2=n
<element>
<id>UMLClass</id>
<coordinates>
<x>441</x>
<y>549</y>
<w>180</w>
<h>117</h>
<x>396</x>
<y>564</y>
<w>240</w>
<h>156</h>
</coordinates>
<panel_attributes>ip_range
--
@ -184,10 +165,10 @@ bg=ORANGE</panel_attributes>
<element>
<id>Relation</id>
<coordinates>
<x>522</x>
<y>459</y>
<w>36</w>
<h>108</h>
<x>504</x>
<y>444</y>
<w>48</w>
<h>144</h>
</coordinates>
<panel_attributes>lt=-
m1=1
@ -198,10 +179,10 @@ m2=n
<element>
<id>UMLClass</id>
<coordinates>
<x>945</x>
<y>306</y>
<w>162</w>
<h>90</h>
<x>1068</x>
<y>240</y>
<w>216</w>
<h>120</h>
</coordinates>
<panel_attributes>route
--
@ -217,10 +198,10 @@ bg=CYAN</panel_attributes>
<element>
<id>UMLClass</id>
<coordinates>
<x>549</x>
<y>135</y>
<w>171</w>
<h>90</h>
<x>540</x>
<y>12</y>
<w>228</w>
<h>120</h>
</coordinates>
<panel_attributes>browser
--
@ -236,10 +217,10 @@ bg=MAGENTA</panel_attributes>
<element>
<id>Relation</id>
<coordinates>
<x>486</x>
<y>216</y>
<w>36</w>
<h>108</h>
<x>456</x>
<y>120</y>
<w>48</w>
<h>144</h>
</coordinates>
<panel_attributes>lt=-
m1=n
@ -250,10 +231,10 @@ m2=1
<element>
<id>UMLClass</id>
<coordinates>
<x>756</x>
<y>135</y>
<w>171</w>
<h>90</h>
<x>816</x>
<y>12</y>
<w>228</w>
<h>120</h>
</coordinates>
<panel_attributes>referer
--
@ -269,10 +250,10 @@ bg=CYAN</panel_attributes>
<element>
<id>Relation</id>
<coordinates>
<x>783</x>
<y>216</y>
<w>36</w>
<h>108</h>
<x>852</x>
<y>120</y>
<w>48</w>
<h>144</h>
</coordinates>
<panel_attributes>lt=-
m1=n
@ -283,10 +264,10 @@ m2=1
<element>
<id>Relation</id>
<coordinates>
<x>864</x>
<y>333</y>
<w>99</w>
<h>45</h>
<x>960</x>
<y>276</y>
<w>132</w>
<h>60</h>
</coordinates>
<panel_attributes>lt=-
m1=n
@ -294,4 +275,70 @@ m2=1
</panel_attributes>
<additional_attributes>10.0;20.0;90.0;20.0</additional_attributes>
</element>
<element>
<id>UMLClass</id>
<coordinates>
<x>1068</x>
<y>564</y>
<w>228</w>
<h>132</h>
</coordinates>
<panel_attributes>country
--
&lt;&lt;PK&gt;&gt;
- country_id: INTEGER
--
- name: TEXT UNIQUE
- code: TEXT UNIQUE
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
</element>
<element>
<id>Relation</id>
<coordinates>
<x>936</x>
<y>708</y>
<w>48</w>
<h>120</h>
</coordinates>
<panel_attributes>lt=-
m1=1
m2=n
</panel_attributes>
<additional_attributes>10.0;80.0;10.0;10.0</additional_attributes>
</element>
<element>
<id>Relation</id>
<coordinates>
<x>1080</x>
<y>684</y>
<w>48</w>
<h>144</h>
</coordinates>
<panel_attributes>lt=-
m1=1
m2=n
</panel_attributes>
<additional_attributes>10.0;10.0;10.0;100.0</additional_attributes>
</element>
<element>
<id>UMLClass</id>
<coordinates>
<x>912</x>
<y>804</y>
<w>228</w>
<h>132</h>
</coordinates>
<panel_attributes>region
--
&lt;&lt;PK&gt;&gt;
- region_id: INTEGER
--
- name: TEXT
- country_id: INTEGER
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
</element>
</diagram>

81
regina/database.py
View File

@ -283,29 +283,42 @@ class Database:
assert(type(country_id_val) == int)
return country_id_val
def get_city_id(self, name, region, country_id) -> int:
def get_region_id(self, name: str, country_id: int) -> int:
"""
get the id of region of name
if not present, insert and return id
"""
name = sanitize(name)
region = sanitize(region)
if not sql_exists(self.cur, "city", [("name", name), ("region", region), ("country_id", country_id)], do_sanitize=False):
self.cur.execute(f"INSERT INTO city (name, region, country_id) VALUES ('{name}', '{region}', '{country_id}')")
cities = sql_select(self.cur, "city", [("name", name), ("region", region), ("country_id", country_id)], do_sanitize=False)
if not sql_exists(self.cur, "region", [("name", name), ("country_id", country_id)], do_sanitize=False):
self.cur.execute(f"INSERT INTO region (name, country_id) VALUES ('{name}', '{country_id}')")
regions = self(f"SELECT region_id FROM region WHERE name = '{name}' AND country_id = '{country_id}'")
if len(regions) > 0:
region_id_val = regions[0][0]
else:
warning(f"get_region_id: Could not get region_id for name='{name}'.")
return 0
assert(type(region_id_val) == int)
return region_id_val
def get_city_id(self, name, region_id: int, country_id:int) -> int:
name = sanitize(name)
if not sql_exists(self.cur, "city", [("name", name), ("region_id", region_id), ("country_id", country_id)], do_sanitize=False):
self.cur.execute(f"INSERT INTO city (name, region_id, country_id) VALUES ('{name}', '{region_id}', '{country_id}')")
cities = sql_select(self.cur, "city", [("name", name), ("region_id", region_id), ("country_id", country_id)], do_sanitize=False)
if len(cities) > 0:
city_id_val = cities[0][0]
else:
warning(f"get_city_id: Could not get city_id for name='{name}', region='{region}' and country_id='{country_id}'.")
warning(f"get_city_id: Could not get city_id for name='{name}', region_id='{region_id}' and country_id='{country_id}'.")
return 0
assert(type(city_id_val) == int)
return city_id_val
def update_geoip_tables(self, geoip_city_csv_path: str):
"""
update the geoip data with the contents of the geoip_city_csv file
Make sure to update the visitor.ip_range_id column for all visitors.
In case something changed, they might point to a different city.
TODO: update teh visitor.ip_range_id column to match (potentially) new city ip range
"""
# indices for the csv
FROM = 0; TO = 1; CODE = 2; COUNTRY = 3; REGION = 4; CITY = 5
@ -333,24 +346,26 @@ class Database:
self.cur.execute(f"DELETE FROM ip_range")
self.cur.execute(f"DELETE FROM city")
self.cur.execute(f"DELETE FROM country")
self.cur.execute(f"DELETE FROM region")
self.conn.commit()
self.cur.execute(f"VACUUM")
# guarantees that unkown city/country will have id 0
self.cur.execute(f"INSERT INTO country (country_id, name, code) VALUES (0, 'Unknown', 'XX') ")
self.cur.execute(f"INSERT INTO city (city_id, name, region) VALUES (0, 'Unknown', 'Unkown') ")
self.cur.execute(f"INSERT INTO region (region_id, name, country_id) VALUES (0, 'Unknown', 0) ")
self.cur.execute(f"INSERT INTO city (city_id, name, region_id, country_id) VALUES (0, 'Unknown', 0, 0) ")
# for combining city ranges into a 'City in <Country>' range
# country_id for the range that was last added (for combining multiple csv rows in one ip_range)
RANGE_DONE = -1
combine_range_country_id = RANGE_DONE
combine_range_country_name = ""
combine_range_city_id = RANGE_DONE
combine_range_low = RANGE_DONE
combine_range_high = RANGE_DONE
def add_range(low, high, city_name, region, country_id):
city_id = self.get_city_id(city_name, region, country_id)
pdebug(f"update_ip_range_id: Adding range for city={city_name:20}, country_id={country_id:3}, low={low:16}, high={high:16}", lvl=2)
get_all = "all" in settings["data-collection"]["get_cities_for_countries"]
def add_range(low, high, city_id):
pdebug(f"update_ip_range_id: Adding range for city={city_id:20}, low={low:16}, high={high:16}", lvl=3)
self.cur.execute(f"INSERT INTO ip_range (low, high, city_id) VALUES ({low}, {high}, {city_id})")
for i, row in enumerate(csv, 1):
# if i % 100 == 0:
@ -365,25 +380,27 @@ class Database:
# make sure country exists
country_id = self.get_country_id(row[COUNTRY], row[CODE])
# only add cities for countries the user is interested in
if row[CODE] in settings["data-collection"]["get_cities_for_countries"]:
add_range(row[FROM], row[TO], row[CITY], row[REGION], country_id)
if get_all or row[CODE] in settings["data-collection"]["get_cities_for_countries"]:
region_id = self.get_region_id(row[REGION], country_id)
city_id = self.get_city_id(row[CITY], region_id, country_id)
else:
# if continuing
if combine_range_country_id != RANGE_DONE:
# if continuing previous range, extend the upper range limit
if combine_range_country_id == country_id:
combine_range_high = row[TO]
else: # new range for country, append
add_range(combine_range_low, combine_range_high, f"City in {combine_range_country_name}", f"Region in {combine_range_country_name}", combine_range_country_id)
combine_range_country_id = RANGE_DONE
# not elif, this has to be executed if previous else was executed
if combine_range_country_id == RANGE_DONE : # currently in new range, combine with later ranges
combine_range_country_id = country_id
combine_range_country_name = row[COUNTRY]
combine_range_low = row[FROM]
region_id = self.get_region_id(f"Region in {row[COUNTRY]}", country_id)
city_id = self.get_city_id(f"City in {row[COUNTRY]}", region_id, country_id)
# if continuing
if combine_range_city_id != RANGE_DONE:
# if continuing previous range, extend the upper range limit
if combine_range_city_id == city_id:
combine_range_high = row[TO]
if combine_range_country_id >= 0: # last range , append
add_range(combine_range_low, combine_range_high, f"City in {combine_range_country_name}", f"Region in {combine_range_country_name}", combine_range_country_id)
else: # new range for city, append
add_range(combine_range_low, combine_range_high, combine_range_city_id)
combine_range_city_id = RANGE_DONE
# not elif, this has to be executed if previous else was executed
if combine_range_city_id == RANGE_DONE : # currently in new range, combine with later ranges
combine_range_city_id = city_id
combine_range_low = row[FROM]
combine_range_high = row[TO]
if combine_range_city_id != RANGE_DONE: # last range , append
add_range(combine_range_low, combine_range_high, combine_range_city_id)
#

10
regina/sql/create_db.sql
View File

@ -59,7 +59,15 @@ CREATE TABLE IF NOT EXISTS ip_range(
CREATE TABLE IF NOT EXISTS city(
city_id INTEGER PRIMARY KEY,
name TEXT,
region TEXT,
region_id INTEGER,
country_id INTEGER,
FOREIGN KEY(region_id) REFERENCES region(region_id),
FOREIGN KEY(country_id) REFERENCES country(country_id)
) STRICT;
CREATE TABLE IF NOT EXISTS region(
region_id INTEGER PRIMARY KEY,
name TEXT,
country_id INTEGER,
FOREIGN KEY(country_id) REFERENCES country(country_id)
) STRICT;