MatthiasQuintern/regina
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added region table to db

Browse Source
This commit is contained in:
matthias@arch 2023-05-18 03:18:49 +02:00
parent b604a92b59
commit d6a033aa5b
3 changed files with 190 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
database.uxf
View File

@ -1,13 +1,13 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?> <?xml version="1.0" encoding="UTF-8" standalone="no"?>
<diagram program="umlet" version="15.1"> <diagram program="umlet" version="15.1">
<zoom_level>9</zoom_level> <zoom_level>12</zoom_level>
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>441</x> <x>396</x>
<y>306</y> <y>240</y>
<w>180</w> <w>240</w>
<h>162</h> <h>216</h>
</coordinates> </coordinates>
<panel_attributes>visitor <panel_attributes>visitor
-- --
@ -27,10 +27,10 @@ bg=MAGENTA</panel_attributes>
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>558</x> <x>552</x>
<y>216</y> <y>120</y>
<w>36</w> <w>48</w>
<h>108</h> <h>144</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=n m1=n
@ -41,10 +41,10 @@ m2=1
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>702</x> <x>744</x>
<y>306</y> <y>240</y>
<w>171</w> <w>228</w>
<h>153</h> <h>204</h>
</coordinates> </coordinates>
<panel_attributes>request <panel_attributes>request
-- --
@ -64,10 +64,10 @@ bg=CYAN</panel_attributes>
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>612</x> <x>624</x>
<y>333</y> <y>276</y>
<w>108</w> <w>144</w>
<h>45</h> <h>60</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=1 m1=1
@ -78,10 +78,10 @@ m2=n
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>333</x> <x>252</x>
<y>135</y> <y>12</y>
<w>180</w> <w>240</w>
<h>90</h> <h>120</h>
</coordinates> </coordinates>
<panel_attributes>platform <panel_attributes>platform
-- --
@ -97,38 +97,19 @@ bg=MAGENTA</panel_attributes>
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>702</x> <x>744</x>
<y>549</y> <y>564</y>
<w>171</w> <w>228</w>
<h>117</h> <h>156</h>
</coordinates> </coordinates>
<panel_attributes>city <panel_attributes>city
-- --
&lt;&lt;PK&gt;&gt; &lt;&lt;PK&gt;&gt;
- city_id: INTEGER - city_id: INTEGER
-- --
- country_id: INTEGER
- name: TEXT - name: TEXT
- region: TEXT
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
</element>
<element>
<id>UMLClass</id>
<coordinates>
<x>945</x>
<y>549</y>
<w>171</w>
<h>99</h>
</coordinates>
<panel_attributes>country
--
&lt;&lt;PK&gt;&gt;
- country_id: INTEGER - country_id: INTEGER
-- - region_id: INTEGER
- name: TEXT UNIQUE
- code: TEXT UNIQUE
style=autoresize style=autoresize
bg=ORANGE</panel_attributes> bg=ORANGE</panel_attributes>
<additional_attributes/> <additional_attributes/>
@ -136,10 +117,10 @@ bg=ORANGE</panel_attributes>
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>864</x> <x>960</x>
<y>567</y> <y>588</y>
<w>99</w> <w>132</w>
<h>45</h> <h>60</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=1 m1=1
@ -150,10 +131,10 @@ m2=n
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>612</x> <x>624</x>
<y>567</y> <y>588</y>
<w>108</w> <w>144</w>
<h>45</h> <h>60</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=1 m1=1
@ -164,10 +145,10 @@ m2=n
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>441</x> <x>396</x>
<y>549</y> <y>564</y>
<w>180</w> <w>240</w>
<h>117</h> <h>156</h>
</coordinates> </coordinates>
<panel_attributes>ip_range <panel_attributes>ip_range
-- --
@ -184,10 +165,10 @@ bg=ORANGE</panel_attributes>
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>522</x> <x>504</x>
<y>459</y> <y>444</y>
<w>36</w> <w>48</w>
<h>108</h> <h>144</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=1 m1=1
@ -198,10 +179,10 @@ m2=n
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>945</x> <x>1068</x>
<y>306</y> <y>240</y>
<w>162</w> <w>216</w>
<h>90</h> <h>120</h>
</coordinates> </coordinates>
<panel_attributes>route <panel_attributes>route
-- --
@ -217,10 +198,10 @@ bg=CYAN</panel_attributes>
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>549</x> <x>540</x>
<y>135</y> <y>12</y>
<w>171</w> <w>228</w>
<h>90</h> <h>120</h>
</coordinates> </coordinates>
<panel_attributes>browser <panel_attributes>browser
-- --
@ -236,10 +217,10 @@ bg=MAGENTA</panel_attributes>
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>486</x> <x>456</x>
<y>216</y> <y>120</y>
<w>36</w> <w>48</w>
<h>108</h> <h>144</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=n m1=n
@ -250,10 +231,10 @@ m2=1
<element> <element>
<id>UMLClass</id> <id>UMLClass</id>
<coordinates> <coordinates>
<x>756</x> <x>816</x>
<y>135</y> <y>12</y>
<w>171</w> <w>228</w>
<h>90</h> <h>120</h>
</coordinates> </coordinates>
<panel_attributes>referer <panel_attributes>referer
-- --
@ -269,10 +250,10 @@ bg=CYAN</panel_attributes>
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>783</x> <x>852</x>
<y>216</y> <y>120</y>
<w>36</w> <w>48</w>
<h>108</h> <h>144</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=n m1=n
@ -283,10 +264,10 @@ m2=1
<element> <element>
<id>Relation</id> <id>Relation</id>
<coordinates> <coordinates>
<x>864</x> <x>960</x>
<y>333</y> <y>276</y>
<w>99</w> <w>132</w>
<h>45</h> <h>60</h>
</coordinates> </coordinates>
<panel_attributes>lt=- <panel_attributes>lt=-
m1=n m1=n
@ -294,4 +275,70 @@ m2=1
</panel_attributes> </panel_attributes>
<additional_attributes>10.0;20.0;90.0;20.0</additional_attributes> <additional_attributes>10.0;20.0;90.0;20.0</additional_attributes>
</element> </element>
<element>
<id>UMLClass</id>
<coordinates>
<x>1068</x>
<y>564</y>
<w>228</w>
<h>132</h>
</coordinates>
<panel_attributes>country
--
&lt;&lt;PK&gt;&gt;
- country_id: INTEGER
--
- name: TEXT UNIQUE
- code: TEXT UNIQUE
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
</element>
<element>
<id>Relation</id>
<coordinates>
<x>936</x>
<y>708</y>
<w>48</w>
<h>120</h>
</coordinates>
<panel_attributes>lt=-
m1=1
m2=n
</panel_attributes>
<additional_attributes>10.0;80.0;10.0;10.0</additional_attributes>
</element>
<element>
<id>Relation</id>
<coordinates>
<x>1080</x>
<y>684</y>
<w>48</w>
<h>144</h>
</coordinates>
<panel_attributes>lt=-
m1=1
m2=n
</panel_attributes>
<additional_attributes>10.0;10.0;10.0;100.0</additional_attributes>
</element>
<element>
<id>UMLClass</id>
<coordinates>
<x>912</x>
<y>804</y>
<w>228</w>
<h>132</h>
</coordinates>
<panel_attributes>region
--
&lt;&lt;PK&gt;&gt;
- region_id: INTEGER
--
- name: TEXT
- country_id: INTEGER
style=autoresize
bg=ORANGE</panel_attributes>
<additional_attributes/>
</element>
</diagram> </diagram>

81
regina/database.py
View File

@ -283,29 +283,42 @@ class Database:
assert(type(country_id_val) == int) assert(type(country_id_val) == int)
return country_id_val return country_id_val
def get_city_id(self, name, region, country_id) -> int: def get_region_id(self, name: str, country_id: int) -> int:
"""
get the id of region of name
if not present, insert and return id
"""
name = sanitize(name) name = sanitize(name)
region = sanitize(region) if not sql_exists(self.cur, "region", [("name", name), ("country_id", country_id)], do_sanitize=False):
if not sql_exists(self.cur, "city", [("name", name), ("region", region), ("country_id", country_id)], do_sanitize=False): self.cur.execute(f"INSERT INTO region (name, country_id) VALUES ('{name}', '{country_id}')")
self.cur.execute(f"INSERT INTO city (name, region, country_id) VALUES ('{name}', '{region}', '{country_id}')") regions = self(f"SELECT region_id FROM region WHERE name = '{name}' AND country_id = '{country_id}'")
cities = sql_select(self.cur, "city", [("name", name), ("region", region), ("country_id", country_id)], do_sanitize=False) if len(regions) > 0:
region_id_val = regions[0][0]
else:
warning(f"get_region_id: Could not get region_id for name='{name}'.")
return 0
assert(type(region_id_val) == int)
return region_id_val
def get_city_id(self, name, region_id: int, country_id:int) -> int:
name = sanitize(name)
if not sql_exists(self.cur, "city", [("name", name), ("region_id", region_id), ("country_id", country_id)], do_sanitize=False):
self.cur.execute(f"INSERT INTO city (name, region_id, country_id) VALUES ('{name}', '{region_id}', '{country_id}')")
cities = sql_select(self.cur, "city", [("name", name), ("region_id", region_id), ("country_id", country_id)], do_sanitize=False)
if len(cities) > 0: if len(cities) > 0:
city_id_val = cities[0][0] city_id_val = cities[0][0]
else: else:
warning(f"get_city_id: Could not get city_id for name='{name}', region='{region}' and country_id='{country_id}'.") warning(f"get_city_id: Could not get city_id for name='{name}', region_id='{region_id}' and country_id='{country_id}'.")
return 0 return 0
assert(type(city_id_val) == int) assert(type(city_id_val) == int)
return city_id_val return city_id_val
def update_geoip_tables(self, geoip_city_csv_path: str): def update_geoip_tables(self, geoip_city_csv_path: str):
""" """
update the geoip data with the contents of the geoip_city_csv file update the geoip data with the contents of the geoip_city_csv file
Make sure to update the visitor.ip_range_id column for all visitors. Make sure to update the visitor.ip_range_id column for all visitors.
In case something changed, they might point to a different city. In case something changed, they might point to a different city.
TODO: update teh visitor.ip_range_id column to match (potentially) new city ip range
""" """
# indices for the csv # indices for the csv
FROM = 0; TO = 1; CODE = 2; COUNTRY = 3; REGION = 4; CITY = 5 FROM = 0; TO = 1; CODE = 2; COUNTRY = 3; REGION = 4; CITY = 5
@ -333,24 +346,26 @@ class Database:
self.cur.execute(f"DELETE FROM ip_range") self.cur.execute(f"DELETE FROM ip_range")
self.cur.execute(f"DELETE FROM city") self.cur.execute(f"DELETE FROM city")
self.cur.execute(f"DELETE FROM country") self.cur.execute(f"DELETE FROM country")
self.cur.execute(f"DELETE FROM region")
self.conn.commit() self.conn.commit()
self.cur.execute(f"VACUUM") self.cur.execute(f"VACUUM")
# guarantees that unkown city/country will have id 0 # guarantees that unkown city/country will have id 0
self.cur.execute(f"INSERT INTO country (country_id, name, code) VALUES (0, 'Unknown', 'XX') ") self.cur.execute(f"INSERT INTO country (country_id, name, code) VALUES (0, 'Unknown', 'XX') ")
self.cur.execute(f"INSERT INTO city (city_id, name, region) VALUES (0, 'Unknown', 'Unkown') ") self.cur.execute(f"INSERT INTO region (region_id, name, country_id) VALUES (0, 'Unknown', 0) ")
self.cur.execute(f"INSERT INTO city (city_id, name, region_id, country_id) VALUES (0, 'Unknown', 0, 0) ")
# for combining city ranges into a 'City in <Country>' range # for combining city ranges into a 'City in <Country>' range
# country_id for the range that was last added (for combining multiple csv rows in one ip_range) # country_id for the range that was last added (for combining multiple csv rows in one ip_range)
RANGE_DONE = -1 RANGE_DONE = -1
combine_range_country_id = RANGE_DONE combine_range_city_id = RANGE_DONE
combine_range_country_name = ""
combine_range_low = RANGE_DONE combine_range_low = RANGE_DONE
combine_range_high = RANGE_DONE combine_range_high = RANGE_DONE
def add_range(low, high, city_name, region, country_id): get_all = "all" in settings["data-collection"]["get_cities_for_countries"]
city_id = self.get_city_id(city_name, region, country_id)
pdebug(f"update_ip_range_id: Adding range for city={city_name:20}, country_id={country_id:3}, low={low:16}, high={high:16}", lvl=2) def add_range(low, high, city_id):
pdebug(f"update_ip_range_id: Adding range for city={city_id:20}, low={low:16}, high={high:16}", lvl=3)
self.cur.execute(f"INSERT INTO ip_range (low, high, city_id) VALUES ({low}, {high}, {city_id})") self.cur.execute(f"INSERT INTO ip_range (low, high, city_id) VALUES ({low}, {high}, {city_id})")
for i, row in enumerate(csv, 1): for i, row in enumerate(csv, 1):
# if i % 100 == 0: # if i % 100 == 0:
@ -365,25 +380,27 @@ class Database:
# make sure country exists # make sure country exists
country_id = self.get_country_id(row[COUNTRY], row[CODE]) country_id = self.get_country_id(row[COUNTRY], row[CODE])
# only add cities for countries the user is interested in # only add cities for countries the user is interested in
if row[CODE] in settings["data-collection"]["get_cities_for_countries"]: if get_all or row[CODE] in settings["data-collection"]["get_cities_for_countries"]:
add_range(row[FROM], row[TO], row[CITY], row[REGION], country_id) region_id = self.get_region_id(row[REGION], country_id)
city_id = self.get_city_id(row[CITY], region_id, country_id)
else: else:
# if continuing region_id = self.get_region_id(f"Region in {row[COUNTRY]}", country_id)
if combine_range_country_id != RANGE_DONE: city_id = self.get_city_id(f"City in {row[COUNTRY]}", region_id, country_id)
# if continuing previous range, extend the upper range limit # if continuing
if combine_range_country_id == country_id: if combine_range_city_id != RANGE_DONE:
combine_range_high = row[TO] # if continuing previous range, extend the upper range limit
else: # new range for country, append if combine_range_city_id == city_id:
add_range(combine_range_low, combine_range_high, f"City in {combine_range_country_name}", f"Region in {combine_range_country_name}", combine_range_country_id)
combine_range_country_id = RANGE_DONE
# not elif, this has to be executed if previous else was executed
if combine_range_country_id == RANGE_DONE : # currently in new range, combine with later ranges
combine_range_country_id = country_id
combine_range_country_name = row[COUNTRY]
combine_range_low = row[FROM]
combine_range_high = row[TO] combine_range_high = row[TO]
if combine_range_country_id >= 0: # last range , append else: # new range for city, append
add_range(combine_range_low, combine_range_high, f"City in {combine_range_country_name}", f"Region in {combine_range_country_name}", combine_range_country_id) add_range(combine_range_low, combine_range_high, combine_range_city_id)
combine_range_city_id = RANGE_DONE
# not elif, this has to be executed if previous else was executed
if combine_range_city_id == RANGE_DONE : # currently in new range, combine with later ranges
combine_range_city_id = city_id
combine_range_low = row[FROM]
combine_range_high = row[TO]
if combine_range_city_id != RANGE_DONE: # last range , append
add_range(combine_range_low, combine_range_high, combine_range_city_id)
# #

10
regina/sql/create_db.sql
View File

@ -59,7 +59,15 @@ CREATE TABLE IF NOT EXISTS ip_range(
CREATE TABLE IF NOT EXISTS city( CREATE TABLE IF NOT EXISTS city(
city_id INTEGER PRIMARY KEY, city_id INTEGER PRIMARY KEY,
name TEXT, name TEXT,
region TEXT, region_id INTEGER,
country_id INTEGER,
FOREIGN KEY(region_id) REFERENCES region(region_id),
FOREIGN KEY(country_id) REFERENCES country(country_id)
) STRICT;
CREATE TABLE IF NOT EXISTS region(
region_id INTEGER PRIMARY KEY,
name TEXT,
country_id INTEGER, country_id INTEGER,
FOREIGN KEY(country_id) REFERENCES country(country_id) FOREIGN KEY(country_id) REFERENCES country(country_id)
) STRICT; ) STRICT;