improved verbosity

regina/main.py

@@ -17,11 +17,9 @@ if __name__ == "__main__":  # make relative imports work as described here: http
 
 from .data_collection.parse_log import parse_log
 from .database import Database
-from .data_visualization import visualize
+from .data_visualization.visualize import visualize
-from .utility.settings_manager import read_settings_file
+from .utility.globals import settings, version, config_dir, data_dir
-from .utility.globals import settings, version
+from .utility.utility import pmessage, pdebug, make_parent_dirs
-from .utility.utility import pmessage
-from .utility.sql_util import sql_tablesize
 
 """
 start regina, launch either collect or visualize
@@ -75,10 +73,9 @@ def error(arg):
     print("Error:", arg)
     exit(1)
 
-
+def main():
-def main2():
     parser = argparse.ArgumentParser(prog="regina")
-    parser.add_argument("--config", "-c",   action="store",         help="path to a config file that specifies all the other parameters", metavar="config-file", required=True)
+    parser.add_argument("--config", "-c",   action="store",         help="path to an alternate config file", metavar="config-file")
     parser.add_argument("--update-geoip",   action="store",         help="path to IP-COUNTRY-REGION-CITY database in csv format", metavar="geoip-csv")
     parser.add_argument("--visualize",      action="store_true",    help="generate the visualization website")
     parser.add_argument("--collect",        action="store_true",    help="fill the database from the nginx access log")
@@ -88,43 +85,57 @@ def main2():
     if not (args.collect or args.visualize or args.update_geoip):
         parser.error("at least one of --visualize, --collect, or --update-geoip is required.")
 
+    if args.config:
         if not path.isfile(args.config):
             parser.error(f"invalid path to configuration file: '{args.config}'")
+        config_path = args.config
 
-    read_settings_file(args.config, settings)
+    else:
-    settings["version"] = version
+        config_path = f"{config_dir}/regina.conf"
+        if not path.isfile(config_path):
+            parser.error(f"missing configuration file: '{config_path}' and no alternative given.")
+
+    try:
+        settings.load(config_path)
+    except ValueError as e:
+        error(f"value error while loading the configuration in '{config_path}':\n\t{e}")
+    except KeyError as e:
+        error(f"key error while loading the configuration in '{config_path}':\n\t{e}")
+    except Exception as e:
+        error(f"while loading the configuration in '{config_path}':\n\t{e}")
+    settings.set("regina", "version", version, allow_new=True)
 
     if args.log_file:
-        settings["access_log"] = args.log_file
+        settings.set("regina", "access_log", args.log_file)
 
-    if not settings["server_name"]:
+    pdebug(f"Settings:\n{settings}", lvl=1)
-        error("'server-name' is missing in the configuration file.")
 
-    if not settings["access_log"]:
+    if not settings["regina"]["database"]:
-        error("'log' is missing in the configuration file.")
+        settings.set(f"regina", "database", f"{data_dir}/{settings['regina']['server_name']}.db")
-
+    db_path = settings["regina"]["database"]
-    if not settings["db"]:
+    make_parent_dirs(db_path)
-        error("'db' is missing in the configuration file.")
+    db = Database(db_path)
-
-    db = Database(settings["db"])
     # if not isfile(settings["db"]):
     #     create_db(settings["db"], settings["filegroups"], settings["locs_and_dirs"], settings["auto_group_filetypes"])
 
     if args.update_geoip:
         if not isfile(args.update_geoip):
-            error(f"Not a file: '{args.update_geoip}'")
+            parser.error(f"invalid path to GeoIP database: '{args.update_geoip}'")
         db.update_geoip_tables(args.update_geoip)
         # update visitors
-        for (visitor_id) in db(f"SELECT visitor_id FROM visitor"):
+        for visitor_id,  in db(f"SELECT visitor_id FROM visitor"):
             db.update_ip_range_id(visitor_id)
     if args.collect:
-        pmessage(f"regina version {version} with server-name '{settings['server_name']}', database '{settings['db']}' and logfile '{settings['access_log']}'")
+        pmessage(f"regina version {version} with server-name '{settings['regina']['server_name']}', database '{db_path}' and logfile '{settings['regina']['access_log']}'")
-        requests = parse_log(settings["access_log"])
+        requests = parse_log(settings['regina']["access_log"])
-        db.add_requests(requests)
+        request_count, visitors_count, new_visitors_count = db.add_requests(requests)
+        if visitors_count > 0: percentage = 100.0*new_visitors_count/visitors_count
+        else: percentage = '--'
+        pmessage(f"Added {request_count} new requests from {visitors_count} different visitors, from which {new_visitors_count} are new ({percentage:2}%)")
+
     if args.visualize:
-        pmessage(f"regina version {version} with server-name '{settings['server_name']}', database '{settings['db']}'")
+        pmessage(f"regina version {version} with server-name '{settings['regina']['server_name']}', database '{db_path}'")
-        if not isfile(settings["db"]): error(f"Invalid database path: '{settings['db']}'")
+        visualize(db)
-        visualize(settings)
 
 if __name__ == '__main__':
-    main2()
+    main()