From 04fdfba36adeeafa385f5e42f1452243197ef1da Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 5 Sep 2025 16:57:39 +0200 Subject: [PATCH 1/8] prototype --- .local_docker_test/Dockerfile | 35 ++++++ .local_docker_test/docker-compose.gh.yml | 21 ++++ .local_docker_test/run-docker-tests.sh | 21 ++++ modelbaker/iliwrapper/ili2dbconfig.py | 30 +++++ modelbaker/iliwrapper/ili2dbtools.py | 10 ++ modelbaker/iliwrapper/ili2dbutils.py | 76 +++++++++++- modelbaker/iliwrapper/iliexecutable.py | 108 +++++++++++++++++- .../testdata/ilimodels/WheresTheAssoc_V1.ili | 25 ++++ 8 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 .local_docker_test/Dockerfile create mode 100644 .local_docker_test/docker-compose.gh.yml create mode 100755 .local_docker_test/run-docker-tests.sh create mode 100644 tests/testdata/ilimodels/WheresTheAssoc_V1.ili diff --git a/.local_docker_test/Dockerfile b/.local_docker_test/Dockerfile new file mode 100644 index 00000000..abe304ad --- /dev/null +++ b/.local_docker_test/Dockerfile @@ -0,0 +1,35 @@ +ARG QGIS_TEST_VERSION=latest +FROM qgis/qgis:${QGIS_TEST_VERSION} + +RUN apt-get update && \ + apt-get -y install openjdk-8-jre \ + && rm -rf /var/lib/apt/lists/* + +# MSSQL: client side +RUN apt-get update +RUN apt-get install -y curl locales postgresql-client libpq-dev +RUN apt-get install -y unixodbc unixodbc-dev odbcinst unixodbc-common libodbcinst2 libodbccr2 libodbc2 libqt5sql5-odbc + +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | tee /etc/apt/sources.list.d/msprod.list +RUN apt-get update +RUN ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools + +# Useful only in 3.22 and 3.28: drop when not need anymore +RUN apt-get update && apt-get -y install python3-pip python3-venv python3-pytest python3-wheel +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv --system-site-packages $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY ./requirements.txt /tmp/ +RUN pip3 install -r /tmp/requirements.txt + +# Avoid sqlcmd termination due to locale -- see https://github.com/Microsoft/mssql-docker/issues/163 +RUN echo "nb_NO.UTF-8 UTF-8" > /etc/locale.gen +RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen +RUN locale-gen +ENV PATH="/usr/local/bin:${PATH}" + +ENV LANG=C.UTF-8 + +WORKDIR / diff --git a/.local_docker_test/docker-compose.gh.yml b/.local_docker_test/docker-compose.gh.yml new file mode 100644 index 00000000..8609d112 --- /dev/null +++ b/.local_docker_test/docker-compose.gh.yml @@ -0,0 +1,21 @@ +version: '3' +services: + mssql: + image: mcr.microsoft.com/mssql/server:2019-CU28-ubuntu-20.04 + environment: + ACCEPT_EULA: Y + SA_PASSWORD: + ports: + - "1433:1433" + + qgis: + build: + context: .. + dockerfile: ./.docker/Dockerfile + args: + QGIS_TEST_VERSION: ${QGIS_TEST_VERSION} + tty: true + volumes: + - ${GITHUB_WORKSPACE}:/usr/src + links: + - mssql diff --git a/.local_docker_test/run-docker-tests.sh b/.local_docker_test/run-docker-tests.sh new file mode 100755 index 00000000..e2aa8422 --- /dev/null +++ b/.local_docker_test/run-docker-tests.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +#*************************************************************************** +# ------------------- +# begin : 2017-08-24 +# git sha : :%H$ +# copyright : (C) 2017 by OPENGIS.ch +# email : info@opengis.ch +#*************************************************************************** +# +#*************************************************************************** +#* * +#* This program is free software; you can redistribute it and/or modify * +#* it under the terms of the GNU General Public License as published by * +#* the Free Software Foundation; either version 2 of the License, or * +#* (at your option) any later version. * +#* * +#*************************************************************************** + +set -e + +/usr/src/tests/testdata/mssql/setup-mssql.sh diff --git a/modelbaker/iliwrapper/ili2dbconfig.py b/modelbaker/iliwrapper/ili2dbconfig.py index 69cc99d4..99bee23d 100644 --- a/modelbaker/iliwrapper/ili2dbconfig.py +++ b/modelbaker/iliwrapper/ili2dbconfig.py @@ -564,3 +564,33 @@ def to_ili2db_args(self, extra_args=[], with_action=True): self.append_args(args, Ili2DbCommandConfiguration.to_ili2db_args(self)) return args + + +class Ili2CCommandConfiguration: + def __init__(self, other=None): + if not isinstance(other, Ili2CCommandConfiguration): + self.base_configuration = BaseConfiguration() + + self.o = "" + self.imdfile = "" + self.ilifile = "" + else: + # We got an 'other' object from which we'll get parameters + self.__dict__ = other.__dict__.copy() + + def append_args(self, args, values): + args += values + + def to_ili2c_args(self): + + args = self.base_configuration.to_ili2db_args(False, False) + + if self.o: + self.append_args(args, ["-o", self.o]) + + if self.imdfile: + self.append_args(args, ["--out", self.imdfile]) + + if self.ilifile: + self.append_args(args, [self.ilifile]) + return args diff --git a/modelbaker/iliwrapper/ili2dbtools.py b/modelbaker/iliwrapper/ili2dbtools.py index 8310a6a9..388be541 100644 --- a/modelbaker/iliwrapper/ili2dbtools.py +++ b/modelbaker/iliwrapper/ili2dbtools.py @@ -55,3 +55,13 @@ def get_tool_url(tool, db_ili_version): ) return "" + + +def get_ili2c_tool_version(): + return "5.6.6" + + +def get_ili2c_tool_url(): + return "https://downloads.interlis.ch/ili2c/ili2c-{version}.zip".format( + version=get_ili2c_tool_version() + ) diff --git a/modelbaker/iliwrapper/ili2dbutils.py b/modelbaker/iliwrapper/ili2dbutils.py index de58d724..1900882e 100644 --- a/modelbaker/iliwrapper/ili2dbutils.py +++ b/modelbaker/iliwrapper/ili2dbutils.py @@ -28,7 +28,12 @@ from ..utils.qt_utils import NetworkError, download_file from .globals import DbIliMode -from .ili2dbtools import get_tool_url, get_tool_version +from .ili2dbtools import ( + get_ili2c_tool_url, + get_ili2c_tool_version, + get_tool_url, + get_tool_version, +) def get_ili2db_bin(tool, db_ili_version, stdout, stderr): @@ -125,6 +130,75 @@ def get_ili2db_bin(tool, db_ili_version, stdout, stderr): return ili2db_file +def get_ili2c_bin(stdout, stderr): + ili_tool_version = get_ili2c_tool_version() + ili_tool_url = get_ili2c_tool_url() + + dir_path = os.path.dirname(os.path.realpath(__file__)) + ili2c_dir = "ili2c-{}".format(ili_tool_version) + + ili2c_file = os.path.join( + dir_path, + "bin", + ili2c_dir, + "ili2c.jar".format(version=ili_tool_version), + ) + + if not os.path.isfile(ili2c_file): + try: + os.makedirs(os.path.join(dir_path, "bin", ili2c_dir), exist_ok=True) + except FileExistsError: + pass + + tmpfile = tempfile.NamedTemporaryFile(suffix=".zip", delete=False) + + stdout.emit( + QCoreApplication.translate( + "ili2dbutils", + "Downloading ili2c version {}…".format(ili_tool_version), + ) + ) + + try: + download_file( + ili_tool_url, + tmpfile.name, + on_progress=lambda received, total: stdout.emit("."), + ) + except NetworkError as e: + stderr.emit( + QCoreApplication.translate( + "ili2dbutils", + 'Could not download ili2c\n\n Error: {error}\n\nFile "{file}" not found. Please download and extract ili2c'.format( + ili2db_url=ili_tool_url, + error=e.msg, + file=ili2c_file, + ), + ) + ) + return None + + try: + with zipfile.ZipFile(tmpfile.name, "r") as z: + z.extractall(os.path.join(dir_path, "bin", ili2c_dir)) + except zipfile.BadZipFile: + # We will realize soon enough that the files were not extracted + pass + + if not os.path.isfile(ili2c_file): + stderr.emit( + QCoreApplication.translate( + "ili2dbutils", + 'File "{file}" not found. Please download and extract ili2c.'.format( + file=ili2c_file, ili2c_url=ili_tool_url + ), + ) + ) + return None + + return ili2c_file + + def get_all_modeldir_in_path(path, lambdafunction=None): all_subdirs = [path[0] for path in os.walk(path)] # include path # Make sure path is included, it can be a special string like `%XTF_DIR` diff --git a/modelbaker/iliwrapper/iliexecutable.py b/modelbaker/iliwrapper/iliexecutable.py index 34f61804..cef344b3 100644 --- a/modelbaker/iliwrapper/iliexecutable.py +++ b/modelbaker/iliwrapper/iliexecutable.py @@ -24,8 +24,8 @@ from ..utils.qt_utils import AbstractQObjectMeta from .ili2dbargs import get_ili2db_args -from .ili2dbconfig import Ili2DbCommandConfiguration -from .ili2dbutils import JavaNotFoundError, get_ili2db_bin, get_java_path +from .ili2dbconfig import Ili2CCommandConfiguration, Ili2DbCommandConfiguration +from .ili2dbutils import JavaNotFoundError, get_ili2c_bin, get_ili2db_bin, get_java_path class IliExecutable(QObject, metaclass=AbstractQObjectMeta): @@ -179,3 +179,107 @@ def stderr_ready(self, proc): def stdout_ready(self, proc): text = bytes(proc.readAllStandardOutput()).decode(self.encoding) self.stdout.emit(text) + + +class IliCompiler(QObject): + SUCCESS = 0 + ERROR = 1000 + ILI2C_NOT_FOUND = 1001 + + stdout = pyqtSignal(str) + stderr = pyqtSignal(str) + process_started = pyqtSignal(str) + process_finished = pyqtSignal(int, int) + cancel_process = pyqtSignal() + + __done_pattern = re.compile(r"Info: \.\.\.([a-zA-Z]+ )?done") + __result = None + + def __init__(self, parent=None): + QObject.__init__(self, parent) + self.filename = None + self.tool = None + self.configuration = self._create_config() + _, self.encoding = locale.getlocale() + + # Lets python try to determine the default locale + if not self.encoding: + _, self.encoding = locale.getdefaultlocale() + + # This might be unset + # (https://stackoverflow.com/questions/1629699/locale-getlocale-problems-on-osx) + if not self.encoding: + self.encoding = "UTF8" + + def _create_config(self) -> Ili2CCommandConfiguration: + """Creates the configuration that will be used by *run* method. + :return: ili2db configuration""" + return Ili2CCommandConfiguration() + + def _args(self, hide_password): + """Gets the list of ili2db arguments from configuration. + + :param bool hide_password: *True* to mask the password, *False* otherwise. + :return: ili2db arguments list. + :rtype: list + """ + self.configuration.to_ili2c_args() + + def _ili2c_jar_arg(self): + ili2c_bin = get_ili2c_bin(self.stdout, self.stderr) + if not ili2c_bin: + return self.ILI2C_NOT_FOUND + return ["-jar", ili2c_bin] + + def _escaped_arg(self, argument): + if '"' in argument: + argument = argument.replace('"', '"""') + if " " in argument: + argument = '"' + argument + '"' + return argument + + def run(self): + proc = QProcess() + self.cancel_process.connect(proc.terminate) + proc.readyReadStandardError.connect( + functools.partial(self.stderr_ready, proc=proc) + ) + proc.readyReadStandardOutput.connect( + functools.partial(self.stdout_ready, proc=proc) + ) + + ili2c_jar_arg = self._ili2c_jar_arg() + if ili2c_jar_arg == self.ILI2C_NOT_FOUND: + return self.ILI2C_NOT_FOUND + args = self._args(False) + java_path = get_java_path(self.configuration.base_configuration) + proc.start(java_path, ili2c_jar_arg + args) + + if not proc.waitForStarted(): + proc = None + + if not proc: + raise JavaNotFoundError() + + self.process_started.emit(self.command_without_password(edited_command)) + + self.__result = self.ERROR + + loop = QEventLoop() + proc.finished.connect(loop.exit) + loop.exec() + + self.process_finished.emit(proc.exitCode(), self.__result) + return self.__result + + def stderr_ready(self, proc): + text = bytes(proc.readAllStandardError()).decode(self.encoding) + + if self.__done_pattern.search(text): + self.__result = self.SUCCESS + + self.stderr.emit(text) + + def stdout_ready(self, proc): + text = bytes(proc.readAllStandardOutput()).decode(self.encoding) + self.stdout.emit(text) diff --git a/tests/testdata/ilimodels/WheresTheAssoc_V1.ili b/tests/testdata/ilimodels/WheresTheAssoc_V1.ili new file mode 100644 index 00000000..9ac06780 --- /dev/null +++ b/tests/testdata/ilimodels/WheresTheAssoc_V1.ili @@ -0,0 +1,25 @@ +INTERLIS 2.3; + +/* Ortsplanung as national model */ +MODEL WheresTheAssoc_V1 (en) AT "https://modelbaker.ch" VERSION "2024-02-07" = + + TOPIC Infrastruktur = + CLASS Item = + Text : TEXT; + END Item; + END Infrastruktur; + + TOPIC Unterhalt = + DEPENDS ON WheresTheAssoc_V1.Infrastruktur; + + CLASS Instandsetzung = + Text : TEXT; + END Instandsetzung; + + ASSOCIATION InstandsetzungItemAssoc = + Item (EXTERNAL) -- {0..*} WheresTheAssoc_V1.Infrastruktur.Item; + Instandsetzung -- {0..*} Instandsetzung; + END InstandsetzungItemAssoc; + + END Unterhalt; +END WheresTheAssoc_V1. From b828718d6474c7cf3c2b7a3efa1b679f2f1056ff Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 5 Sep 2025 21:56:34 +0200 Subject: [PATCH 2/8] fix --- modelbaker/iliwrapper/ili2dbconfig.py | 6 +++--- modelbaker/iliwrapper/iliexecutable.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modelbaker/iliwrapper/ili2dbconfig.py b/modelbaker/iliwrapper/ili2dbconfig.py index 99bee23d..f4fa677f 100644 --- a/modelbaker/iliwrapper/ili2dbconfig.py +++ b/modelbaker/iliwrapper/ili2dbconfig.py @@ -571,7 +571,7 @@ def __init__(self, other=None): if not isinstance(other, Ili2CCommandConfiguration): self.base_configuration = BaseConfiguration() - self.o = "" + self.oIMD16 = True self.imdfile = "" self.ilifile = "" else: @@ -585,8 +585,8 @@ def to_ili2c_args(self): args = self.base_configuration.to_ili2db_args(False, False) - if self.o: - self.append_args(args, ["-o", self.o]) + if self.oIMD16: + self.append_args(args, ["-oIMD16"]) if self.imdfile: self.append_args(args, ["--out", self.imdfile]) diff --git a/modelbaker/iliwrapper/iliexecutable.py b/modelbaker/iliwrapper/iliexecutable.py index cef344b3..4d1e5fa0 100644 --- a/modelbaker/iliwrapper/iliexecutable.py +++ b/modelbaker/iliwrapper/iliexecutable.py @@ -223,7 +223,7 @@ def _args(self, hide_password): :return: ili2db arguments list. :rtype: list """ - self.configuration.to_ili2c_args() + return self.configuration.to_ili2c_args() def _ili2c_jar_arg(self): ili2c_bin = get_ili2c_bin(self.stdout, self.stderr) @@ -253,6 +253,7 @@ def run(self): return self.ILI2C_NOT_FOUND args = self._args(False) java_path = get_java_path(self.configuration.base_configuration) + proc.start(java_path, ili2c_jar_arg + args) if not proc.waitForStarted(): @@ -261,7 +262,7 @@ def run(self): if not proc: raise JavaNotFoundError() - self.process_started.emit(self.command_without_password(edited_command)) + # self.process_started.emit(self.command_without_password(edited_command)) self.__result = self.ERROR From 08c5fcd3905833725d7397f23c056ddb4d956ae0 Mon Sep 17 00:00:00 2001 From: signedav Date: Sun, 7 Sep 2025 23:06:45 +0200 Subject: [PATCH 3/8] tinkerlis --- modelbaker/iliwrapper/ilicache.py | 112 +++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 24 deletions(-) diff --git a/modelbaker/iliwrapper/ilicache.py b/modelbaker/iliwrapper/ilicache.py index 37856a15..531e654f 100644 --- a/modelbaker/iliwrapper/ilicache.py +++ b/modelbaker/iliwrapper/ilicache.py @@ -890,31 +890,34 @@ def refresh(self): # collect local files netloc = "local_files" repo_files = list() - for file_path_id in [ - file_id for file_id in self.file_ids if file_id[0:5] == "file:" - ]: - toppingfile = dict() - toppingfile["id"] = file_path_id - toppingfile["version"] = None - toppingfile["owner"] = None - toppingfile["repository"] = netloc - toppingfile["url"] = None - toppingfile["relative_file_path"] = file_path_id[5:] - toppingfile["local_file_path"] = ( - file_path_id[5:] - if os.path.isabs(file_path_id[5:]) - else os.path.join(self.tool_dir, file_path_id[5:]) - ) - if os.path.exists(toppingfile["local_file_path"]): - self.file_download_succeeded.emit( - file_path_id, toppingfile["local_file_path"] - ) - else: - self.file_download_failed.emit( - file_path_id, - self.tr("Could not find local file {}").format(file_path_id[5:]), + if self.file_ids: + for file_path_id in [ + file_id for file_id in self.file_ids if file_id[0:5] == "file:" + ]: + toppingfile = dict() + toppingfile["id"] = file_path_id + toppingfile["version"] = None + toppingfile["owner"] = None + toppingfile["repository"] = netloc + toppingfile["url"] = None + toppingfile["relative_file_path"] = file_path_id[5:] + toppingfile["local_file_path"] = ( + file_path_id[5:] + if os.path.isabs(file_path_id[5:]) + else os.path.join(self.tool_dir, file_path_id[5:]) ) - repo_files.append(toppingfile) + if os.path.exists(toppingfile["local_file_path"]): + self.file_download_succeeded.emit( + file_path_id, toppingfile["local_file_path"] + ) + else: + self.file_download_failed.emit( + file_path_id, + self.tr("Could not find local file {}").format( + file_path_id[5:] + ), + ) + repo_files.append(toppingfile) self.repositories[netloc] = repo_files self.set_repositories_to_model() @@ -1050,3 +1053,64 @@ def set_repositories(self, repositories): ids.append(toppingfile["id"]) self.appendRow(item) + + +class IliModelFileCache(IliToppingFileCache): + + CACHE_PATH = os.path.expanduser("~/.ilimodelsfilescache") + + def __init__(self, configuration, model_name_list=None): + IliToppingFileCache.__init__(self, configuration, model_name_list) + self.information_file = "ilimodels.xml" + self.model_name_list = model_name_list + self.ilifilelist = [] + + def _process_informationfile(self, file, netloc, url): + """ + Parses ilimodels.xml provided in ``file`` and updates the local repositories cache. + """ + + try: + root = ET.parse(file).getroot() + except ET.ParseError as e: + QgsMessageLog.logMessage( + self.tr( + "Could not parse ilimodels file `{file}` ({exception})".format( + file=file, exception=str(e) + ) + ), + self.tr("modelbaker"), + ) + return + + for repo in root.iter( + "{http://www.interlis.ch/INTERLIS2.3}IliRepository09.RepositoryIndex" + ): + for model_metadata in repo.findall( + "ili23:IliRepository09.RepositoryIndex.ModelMetadata", self.ns + ): + name = self.get_element_text( + element=model_metadata.find("ili23:Name", self.ns) + ) + if name in self.model_name_list: + file = self.get_element_text( + element=model_metadata.find("ili23:File", self.ns) + ) + + self.ilifilelist.append(self.download_file(netloc, url, file, name)) + + for repo in root.iter( + "{http://www.interlis.ch/INTERLIS2.3}IliRepository20.RepositoryIndex" + ): + for model_metadata in repo.findall( + "ili23:IliRepository20.RepositoryIndex.ModelMetadata", self.ns + ): + name = self.get_element_text( + element=model_metadata.find("ili23:Name", self.ns) + ) + if name in self.model_name_list: + file = self.get_element_text( + element=model_metadata.find("ili23:File", self.ns) + ) + + self.ilifilelist.append(self.download_file(netloc, url, file, name)) From 2126ba8ac6693cad486e95ded0768115d5291ee9 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 8 Sep 2025 10:13:47 +0200 Subject: [PATCH 4/8] first working base --- modelbaker/iliwrapper/iliexecutable.py | 27 ++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/modelbaker/iliwrapper/iliexecutable.py b/modelbaker/iliwrapper/iliexecutable.py index 4d1e5fa0..37840597 100644 --- a/modelbaker/iliwrapper/iliexecutable.py +++ b/modelbaker/iliwrapper/iliexecutable.py @@ -182,7 +182,7 @@ def stdout_ready(self, proc): class IliCompiler(QObject): - SUCCESS = 0 + SUCCESS = 1000 # not sure why it returns 1000 but whatever ERROR = 1000 ILI2C_NOT_FOUND = 1001 @@ -216,7 +216,7 @@ def _create_config(self) -> Ili2CCommandConfiguration: :return: ili2db configuration""" return Ili2CCommandConfiguration() - def _args(self, hide_password): + def _args(self): """Gets the list of ili2db arguments from configuration. :param bool hide_password: *True* to mask the password, *False* otherwise. @@ -238,6 +238,25 @@ def _escaped_arg(self, argument): argument = '"' + argument + '"' return argument + def command(self): + ili2c_jar_arg = self._ili2c_jar_arg() + if ili2c_jar_arg == self.ILI2C_NOT_FOUND: + return "ili2c tool not found!" + + args = self._args() + java_path = self._escaped_arg( + get_java_path(self.configuration.base_configuration) + ) + command_args = ili2c_jar_arg + args + + valid_args = [] + for command_arg in command_args: + valid_args.append(self._escaped_arg(command_arg)) + + command = java_path + " " + " ".join(valid_args) + + return command + def run(self): proc = QProcess() self.cancel_process.connect(proc.terminate) @@ -251,7 +270,7 @@ def run(self): ili2c_jar_arg = self._ili2c_jar_arg() if ili2c_jar_arg == self.ILI2C_NOT_FOUND: return self.ILI2C_NOT_FOUND - args = self._args(False) + args = self._args() java_path = get_java_path(self.configuration.base_configuration) proc.start(java_path, ili2c_jar_arg + args) @@ -262,7 +281,7 @@ def run(self): if not proc: raise JavaNotFoundError() - # self.process_started.emit(self.command_without_password(edited_command)) + self.process_started.emit(self.command()) self.__result = self.ERROR From b1536a7268d96523a08bdea43e88dada9d482fc2 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 8 Sep 2025 15:50:05 +0200 Subject: [PATCH 5/8] modularize and project generate get enum fields --- modelbaker/dataobjects/fields.py | 5 ++ modelbaker/dataobjects/project.py | 65 ++++++++++++++ modelbaker/generator/generator.py | 24 +++--- modelbaker/pythonizer/pythonizer.py | 128 ++++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 14 deletions(-) create mode 100644 modelbaker/pythonizer/pythonizer.py diff --git a/modelbaker/dataobjects/fields.py b/modelbaker/dataobjects/fields.py index 1a695f28..08ffe804 100644 --- a/modelbaker/dataobjects/fields.py +++ b/modelbaker/dataobjects/fields.py @@ -36,17 +36,22 @@ def __init__(self, name: str) -> None: self.default_value_expression = None self.enum_domain = None self.oid_domain = None + self.ili_name = None def dump(self) -> dict: definition = dict() if self.alias: definition["alias"] = self.alias + if self.ili_name: + definition["ili_name"] = self.ili_name return definition def load(self, definition: dict) -> None: if "alias" in definition: self.alias = definition["alias"] + if "ili_name" in definition: + self.ili_name = definition["ili_name"] def create(self, layer: Layer) -> None: field_idx = layer.layer.fields().indexOf(self.name) diff --git a/modelbaker/dataobjects/project.py b/modelbaker/dataobjects/project.py index 46afb7cb..a42e890e 100644 --- a/modelbaker/dataobjects/project.py +++ b/modelbaker/dataobjects/project.py @@ -34,7 +34,14 @@ ) from qgis.PyQt.QtCore import QObject, pyqtSignal from qgis.PyQt.QtXml import QDomDocument +from QgisModelBaker.libs.modelbaker.libs.ili2py.interfaces.interlis.interlis_24.ilismeta.ilismeta16_2022_10_10.enum_type_type import ( + EnumTypeType, +) +from QgisModelBaker.libs.modelbaker.libs.ili2py.writers.py.interlis23.python import ( + Enumeration, +) +from ..pythonizer.pythonizer import Pythonizer from ..utils.globals import LogLevel, OptimizeStrategy, default_log_function from .layers import Layer from .legend import LegendGroup @@ -52,6 +59,9 @@ def __init__( evaluate_default_values: bool = True, context: dict[str, str] = {}, optimize_strategy: OptimizeStrategy = OptimizeStrategy.NONE, + pythonize_enums=None, + configuration=None, + models=[], log_function=None, ) -> None: QObject.__init__(self) @@ -68,6 +78,9 @@ def __init__( self.mapthemes = {} self.context = context self.optimize_strategy = optimize_strategy + self.pythonize_enums = pythonize_enums + self.configuration = configuration + self.models = models self.log_function = log_function if not log_function: @@ -315,6 +328,58 @@ def create( self.tr("The minimal selection is 1"), ) + if self.pythonize_enums: + self.log_function( + f"Let's get all the enums via ili2py and write them into Value Maps... Just for fun.", + LogLevel.INFO, + ) + pythonizer = Pythonizer() + model_files = pythonizer.model_files( + self.configuration.base_configuration, self.models + ) + self.log_function( + f"Received modelfiles {model_files} for models {self.models}", + LogLevel.INFO, + ) + if model_files: + result, imd_file = pythonizer.compile( + self.configuration.base_configuration, model_files[0] + ) + if result: + self.log_function( + f"Having a nice imd {imd_file}", + LogLevel.INFO, + ) + index, _ = pythonizer.pythonize(imd_file) + if index: + self.log_function( + "Having a proper index", + LogLevel.INFO, + ) + for layer in self.layers: + for field in layer.fields: + # if field.enum_domain: + if not field.ili_name: + continue + imd_field_object = index.index.get(field.ili_name) + if not hasattr(imd_field_object, "type_value"): + continue + imd_field_type_oid = imd_field_object.type_value.ref + imd_field_type_object = index.index.get(imd_field_type_oid) + + if isinstance(imd_field_type_object, EnumTypeType): + self.log_function( + f"Having a field with an enum {field.ili_name}", + LogLevel.INFO, + ) + + enum_object = Enumeration.from_imd( + imd_field_type_object, index + ) + self.log_function( + f"Enum value: {enum_object.values}", + LogLevel.INFO, + ) for layer in self.layers: if layer.layer.type() == QgsMapLayer.LayerType.VectorLayer: # even when a style will be loaded we create the form because not sure if the style contains form settngs diff --git a/modelbaker/generator/generator.py b/modelbaker/generator/generator.py index 2391bc8b..9f9c9ebb 100644 --- a/modelbaker/generator/generator.py +++ b/modelbaker/generator/generator.py @@ -332,7 +332,11 @@ def layers(self, filter_layer_list: list = []) -> list[Layer]: re_iliname = re.compile(r".*\.(.*)$") for fielddef in fields_info: column_name = fielddef["column_name"] - + ili_name = fully_qualified_name = ( + fielddef["fully_qualified_name"] + if "fully_qualified_name" in fielddef + else None + ) # If raw_naming is True, the fieldname should be the columnname # Otherwise get field name in this order: # - translation if exists, @@ -345,22 +349,14 @@ def layers(self, filter_layer_list: list = []) -> list[Layer]: alias = fielddef.get("column_tr", None) if not alias: alias = fielddef.get("column_alias", None) - if not alias: - fully_qualified_name = ( - fielddef["fully_qualified_name"] - if "fully_qualified_name" in fielddef - else None - ) - m = ( - re_iliname.match(fully_qualified_name) - if fully_qualified_name - else None - ) - if m: - alias = m.group(1) + if ili_name: + m = re_iliname.match(ili_name) if ili_name else None + if m: + alias = m.group(1) field = Field(column_name) + field.ili_name = ili_name field.alias = alias # Should we hide the field? diff --git a/modelbaker/pythonizer/pythonizer.py b/modelbaker/pythonizer/pythonizer.py new file mode 100644 index 00000000..f3bc2441 --- /dev/null +++ b/modelbaker/pythonizer/pythonizer.py @@ -0,0 +1,128 @@ +import datetime +import os + +from qgis.core import Qgis +from qgis.PyQt.QtCore import QEventLoop, QObject, QStandardPaths, QTimer +from QgisModelBaker.libs.modelbaker.iliwrapper import iliexecutable +from QgisModelBaker.libs.modelbaker.iliwrapper.ili2dbconfig import ( + Ili2CCommandConfiguration, +) +from QgisModelBaker.libs.modelbaker.iliwrapper.ili2dbutils import JavaNotFoundError +from QgisModelBaker.libs.modelbaker.iliwrapper.ilicache import IliModelFileCache +from QgisModelBaker.libs.modelbaker.libs.ili2py.mappers.helpers import Index +from QgisModelBaker.libs.modelbaker.libs.ili2py.readers.interlis_24.ilismeta16.xsdata import ( + Imd16Reader, +) +from QgisModelBaker.libs.modelbaker.libs.ili2py.writers.py import Library + +from ..utils.globals import default_log_function + + +class Pythonizer(QObject): + def __init__(self, log_function=None) -> None: + QObject.__init__(self) + + self.log_function = log_function if log_function else default_log_function + + if not log_function: + self.log_function = default_log_function + + def compile(self, base_configuration, ili_file): + compiler = iliexecutable.IliCompiler() + + configuration = Ili2CCommandConfiguration() + configuration.base_configuration = base_configuration + configuration.ilifile = ili_file + configuration.imdfile = os.path.join( + QStandardPaths.writableLocation( + QStandardPaths.StandardLocation.TempLocation + ), + "temp_imd_{:%Y%m%d%H%M%S%f}.imd".format(datetime.datetime.now()), + ) + + compiler.configuration = configuration + + compiler.stdout.connect(self.on_ili_stdout) + compiler.stderr.connect(self.on_ili_stderr) + compiler.process_started.connect(self.on_ili_process_started) + compiler.process_finished.connect(self.on_ili_process_finished) + result = True + + try: + compiler_result = compiler.run() + if compiler_result != compiler.SUCCESS: + result = False + except JavaNotFoundError as e: + self.log_function( + self.tr("Java not found error: {}").format(e.error_string), + Qgis.MessageLevel.Warning, + ) + result = False + + return result, compiler.configuration.imdfile + + def pythonize(self, imd_file): + reader = Imd16Reader() + metamodel = reader.read(imd_file) + index = Index(metamodel.datasection) + library_name = index.types_bucket["Model"][-1].name + library = Library.from_imd(metamodel.datasection.ModelData, index, library_name) + return index, library + + def model_files(self, base_configuration, model_list): + model_file_cache = IliModelFileCache(base_configuration, model_list) + # we wait for the download or we timeout after 30 seconds and we apply what we have + loop = QEventLoop() + model_file_cache.download_finished_and_model_fresh.connect(lambda: loop.quit()) + timer = QTimer() + timer.setSingleShot(True) + timer.timeout.connect(lambda: loop.quit()) + timer.start(10000) + + model_file_cache.refresh() + self.log_function(self.tr("- - Downloading…")) + + # we wait for the download_finished_and_model_fresh signal, because even when the files are local, it should only continue when both is ready + loop.exec() + + if len(model_file_cache.downloaded_files) == len(model_list): + self.log_function(self.tr("- - All model files")) + else: + missing_file_ids = model_list + for downloaded_file_id in model_file_cache.downloaded_files: + if downloaded_file_id in missing_file_ids: + missing_file_ids.remove(downloaded_file_id) + try: + self.log_function( + self.tr( + "- - Some model files where not successfully downloaded: {}" + ).format(" ".join(missing_file_ids)) + ) + except Exception: + pass + + return model_file_cache.ilifilelist + + def on_ili_stdout(self, message): + lines = message.strip().split("\n") + for line in lines: + text = f"ili2c: {line}" + self.log_function(text, Qgis.MessageLevel.Info) + + def on_ili_stderr(self, message): + lines = message.strip().split("\n") + for line in lines: + text = f"ili2c: {line}" + self.log_function(text, Qgis.MessageLevel.Critical) + + def on_ili_process_started(self, command): + text = f"ili2c: {command}" + self.log_function(text, Qgis.MessageLevel.Info) + + def on_ili_process_finished(self, exit_code, result): + if exit_code == 0: + text = f"ili2c: Successfully performed command." + self.log_function(text, Qgis.MessageLevel.Info) + else: + text = f"ili2c: Finished with errors: {result}" + self.log_function(text, Qgis.MessageLevel.Critical) From 96214fb8750657875c0ccfdf4934b6227347fc24 Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 12 Sep 2025 13:24:50 +0200 Subject: [PATCH 6/8] get models from db and create value map from enums --- modelbaker/dataobjects/project.py | 20 +++++++++++++++-- modelbaker/dbconnector/db_connector.py | 6 ++++++ modelbaker/pythonizer/pythonizer.py | 30 ++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/modelbaker/dataobjects/project.py b/modelbaker/dataobjects/project.py index a42e890e..a7e9a46a 100644 --- a/modelbaker/dataobjects/project.py +++ b/modelbaker/dataobjects/project.py @@ -194,6 +194,9 @@ def create( and referenced_layer.is_enum and not referenced_layer.display_expression ): + if self.pythonize_enums: + continue + editor_widget_setup = QgsEditorWidgetSetup( "ValueRelation", { @@ -219,6 +222,9 @@ def create( }, ) elif referenced_layer and referenced_layer.is_domain: + + if self.pythonize_enums: + continue editor_widget_setup = QgsEditorWidgetSetup( "RelationReference", { @@ -334,8 +340,8 @@ def create( LogLevel.INFO, ) pythonizer = Pythonizer() - model_files = pythonizer.model_files( - self.configuration.base_configuration, self.models + model_files = pythonizer.model_files_generated_from_db( + self.configuration, self.models ) self.log_function( f"Received modelfiles {model_files} for models {self.models}", @@ -358,6 +364,7 @@ def create( ) for layer in self.layers: for field in layer.fields: + # if field.enum_domain: if not field.ili_name: continue @@ -380,6 +387,15 @@ def create( f"Enum value: {enum_object.values}", LogLevel.INFO, ) + widget = "ValueMap" + config = {"map": {}} + config["map"] = [ + {val: val} for val in enum_object.values + ] + setup = QgsEditorWidgetSetup(widget, config) + + field_idx = layer.layer.fields().indexOf(field.name) + layer.layer.setEditorWidgetSetup(field_idx, setup) for layer in self.layers: if layer.layer.type() == QgsMapLayer.LayerType.VectorLayer: # even when a style will be loaded we create the form because not sure if the style contains form settngs diff --git a/modelbaker/dbconnector/db_connector.py b/modelbaker/dbconnector/db_connector.py index 1fef556e..be027467 100644 --- a/modelbaker/dbconnector/db_connector.py +++ b/modelbaker/dbconnector/db_connector.py @@ -286,6 +286,12 @@ def get_models(self): """ return {} + def get_model_content(self, model): + """ + Returns content of the model + """ + return {} + def ili_version(self): """ Returns the version of the ili2db application that was used to create the schema. diff --git a/modelbaker/pythonizer/pythonizer.py b/modelbaker/pythonizer/pythonizer.py index f3bc2441..f77bc1d7 100644 --- a/modelbaker/pythonizer/pythonizer.py +++ b/modelbaker/pythonizer/pythonizer.py @@ -1,8 +1,9 @@ import datetime import os +import QgisModelBaker.libs.modelbaker.utils.db_utils as db_utils from qgis.core import Qgis -from qgis.PyQt.QtCore import QEventLoop, QObject, QStandardPaths, QTimer +from qgis.PyQt.QtCore import QEventLoop, QFile, QObject, QStandardPaths, QTimer from QgisModelBaker.libs.modelbaker.iliwrapper import iliexecutable from QgisModelBaker.libs.modelbaker.iliwrapper.ili2dbconfig import ( Ili2CCommandConfiguration, @@ -69,7 +70,32 @@ def pythonize(self, imd_file): library = Library.from_imd(metamodel.datasection.ModelData, index, library_name) return index, library - def model_files(self, base_configuration, model_list): + def model_files_generated_from_db(self, configuration, model_list): + model_files = [] + # this could be improved i guess, we already have the models read from the same function. but yes. poc etc. + db_connector = db_utils.get_db_connector(configuration) + db_connector.get_models() + model_records = db_connector.get_models() + for record in model_records: + name = record["modelname"].split("{")[0] + if name in model_list: + modelfilepath = os.path.join( + QStandardPaths.writableLocation( + QStandardPaths.StandardLocation.TempLocation + ), + "temp_{}_{:%Y%m%d%H%M%S%f}.ili".format( + name, datetime.datetime.now() + ), + ) + file = QFile(modelfilepath) + if file.open(QFile.OpenModeFlag.WriteOnly): + file.write(record["content"].encode("utf-8")) + file.close() + model_files.append(modelfilepath) + print(modelfilepath) + return model_files + + def model_files_from_repo(self, base_configuration, model_list): model_file_cache = IliModelFileCache(base_configuration, model_list) # we wait for the download or we timeout after 30 seconds and we apply what we have loop = QEventLoop() From d428fb16e1628e113837fc2689fad99146251547 Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 12 Sep 2025 17:29:49 +0200 Subject: [PATCH 7/8] pythonizer --- modelbaker/dataobjects/project.py | 2 +- modelbaker/pythonizer/pythonizer.py | 4 +++- .../ilimodels/InfrastrukturBasketOID_V1.ili | 20 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/testdata/ilimodels/InfrastrukturBasketOID_V1.ili diff --git a/modelbaker/dataobjects/project.py b/modelbaker/dataobjects/project.py index a7e9a46a..d5873428 100644 --- a/modelbaker/dataobjects/project.py +++ b/modelbaker/dataobjects/project.py @@ -37,7 +37,7 @@ from QgisModelBaker.libs.modelbaker.libs.ili2py.interfaces.interlis.interlis_24.ilismeta.ilismeta16_2022_10_10.enum_type_type import ( EnumTypeType, ) -from QgisModelBaker.libs.modelbaker.libs.ili2py.writers.py.interlis23.python import ( +from QgisModelBaker.libs.modelbaker.libs.ili2py.writers.py.python_structure import ( Enumeration, ) diff --git a/modelbaker/pythonizer/pythonizer.py b/modelbaker/pythonizer/pythonizer.py index f77bc1d7..83462ada 100644 --- a/modelbaker/pythonizer/pythonizer.py +++ b/modelbaker/pythonizer/pythonizer.py @@ -14,7 +14,9 @@ from QgisModelBaker.libs.modelbaker.libs.ili2py.readers.interlis_24.ilismeta16.xsdata import ( Imd16Reader, ) -from QgisModelBaker.libs.modelbaker.libs.ili2py.writers.py import Library +from QgisModelBaker.libs.modelbaker.libs.ili2py.writers.py.python_structure import ( + Library, +) from ..utils.globals import default_log_function diff --git a/tests/testdata/ilimodels/InfrastrukturBasketOID_V1.ili b/tests/testdata/ilimodels/InfrastrukturBasketOID_V1.ili new file mode 100644 index 00000000..eb9af4ac --- /dev/null +++ b/tests/testdata/ilimodels/InfrastrukturBasketOID_V1.ili @@ -0,0 +1,20 @@ +INTERLIS 2.3; + +/* Ortsplanung as national model */ +MODEL Infrastruktur_V1 (en) AT "https://modelbaker.ch" VERSION "2023-03-29" = + IMPORTS GeometryCHLV95_V1; + + DOMAIN + CHLine = POLYLINE WITH (STRAIGHTS) VERTEX GeometryCHLV95_V1.Coord2; + CHSurface = SURFACE WITH (STRAIGHTS) VERTEX GeometryCHLV95_V1.Coord2 WITHOUT OVERLAPS > 0.001; + + TOPIC Strassen = + BASKET OID AS INTERLIS.UUIDOID; + OID AS INTERLIS.UUIDOID; + CLASS Strasse = + Name : MANDATORY TEXT*99; + Geometrie : MANDATORY Infrastruktur_V1.CHLine; + END Strasse; + END Strassen; + +END Infrastruktur_V1. From e5dfbfa1f70a969e158672ef51a6e190bc7a2888 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 3 Nov 2025 16:30:25 +0100 Subject: [PATCH 8/8] simply save ili path in iliCache model --- modelbaker/iliwrapper/ilicache.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modelbaker/iliwrapper/ilicache.py b/modelbaker/iliwrapper/ilicache.py index 531e654f..cf8c8013 100644 --- a/modelbaker/iliwrapper/ilicache.py +++ b/modelbaker/iliwrapper/ilicache.py @@ -245,6 +245,9 @@ def _process_informationfile(self, file, netloc, url): model["version"] = self.get_element_text( model_metadata.find("ili23:Version", self.ns) ) + model["file"] = self.get_element_text( + model_metadata.find("ili23:File", self.ns) + ) model["repository"] = netloc repo_models.append(model) @@ -262,6 +265,9 @@ def _process_informationfile(self, file, netloc, url): model["version"] = self.get_element_text( model_metadata.find("ili23:Version", self.ns) ) + model["file"] = self.get_element_text( + model_metadata.find("ili23:File", self.ns) + ) model["repository"] = netloc repo_models.append(model) @@ -375,6 +381,7 @@ class IliModelItemModel(QStandardItemModel): class Roles(Enum): ILIREPO = Qt.ItemDataRole.UserRole + 1 VERSION = Qt.ItemDataRole.UserRole + 2 + FILE = Qt.ItemDataRole.UserRole + 3 def __int__(self): return self.value @@ -400,6 +407,7 @@ def set_repositories(self, repositories): ) # considered in completer item.setData(model["repository"], int(IliModelItemModel.Roles.ILIREPO)) item.setData(model["version"], int(IliModelItemModel.Roles.VERSION)) + item.setData(model["file"], int(IliModelItemModel.Roles.FILE)) names.append(model["name"]) self.appendRow(item)