Implement ci target for Makefile to reproduce github CI actions & artifacts (#1769)

* Implement CI target in Makefile to emulate github CI actions & artifacts

* Improve filter for metadata

* metadata.py: update usage output for wrong number of input arguments / code review

* metadata.py: remove excessive checks for the second input argument / code review

* metadata.py: remove hard-coded model for multi-lang builds in ModelName argument processing / code review

* metadata.py: remove hard-coded models for multi-lang builds in file name pattern processing / code review

* metadata.py: update usage output to remove ambiguity about json extension for output file

* metadata.py: unify new lines style formatting

* metadata.py: sort the list of processing files in alphanumeric order before looping through them to get the same lang order on every generation in every json output file

3 files changed, 70 insertions, 15 deletions

diff --git a/Makefile b/Makefile
index 3c664950..3ad91515 100644
--- a/Makefile
+++ b/Makefile
@@ -51,6 +51,14 @@ DOCKER_CMD=$(DOCKER_BIN)  -f $(DOCKER_YML)  run  --rm  builder
 # MkDocs config

 MKDOCS_YML=$(CURDIR)/scripts/IronOS-mkdocs.yml

 

+# supported models

+MODELS=TS100 TS80 TS80P Pinecil MHP30 Pinecilv2 S60 TS101 # target names & dir names

+MODELS_ML=Pinecil  Pinecilv2 # target names

+MODELS_MULTILANG=Pinecil_multi-lang  Pinecilv2_multi-lang # dir names

+

+# zip command (to pack artifacts)

+ZIP=zip -q -j -r

+

 

 ### targets

 

@@ -91,7 +99,7 @@ list:
 	@echo

 	@echo "Full list of current supported IDs:"

 	@echo "  * LANG_ID: $(shell echo "`ls Translations/ | grep -e "^translation_.*.json$$" | sed -e 's,^translation_,,g; s,\.json$$,,g; ' | tr '\n' ' '`")"

-	@echo "  * MODEL_ID: TS100 TS101 TS80 TS80P MHP30 Pinecil Pinecilv2 S60"

+	@echo "  * MODEL_ID: $(MODELS)"

 	@echo

 	@echo "For example, to make a local build of IronOS firmware for TS100 with English language, just type:"

 	@echo

@@ -171,11 +179,41 @@ build-all:
 	@chmod  0777  $(OUT_DIR)

 	cd  source  &&  bash  ./build.sh

 	@echo "All Firmware built"

-	@cp  -r  $(OUT_HEX)/*.bin  $(OUT_DIR)

-	@cp  -r  $(OUT_HEX)/*.hex  $(OUT_DIR)

-	@cp  -r  $(OUT_HEX)/*.dfu  $(OUT_DIR)

+	@for model in $(MODELS); do \

+		mkdir  -p  $(OUT_DIR)/$${model} ; \

+		cp  -r  $(OUT_HEX)/$${model}_*.bin  $(OUT_DIR)/$${model}/ ; \

+		cp  -r  $(OUT_HEX)/$${model}_*.hex  $(OUT_DIR)/$${model}/ ; \

+		cp  -r  $(OUT_HEX)/$${model}_*.dfu  $(OUT_DIR)/$${model}/ ; \

+	done;

 	@echo "Resulting output directory: $(OUT_DIR)"

 

+# target to build multilang supported builds for Pinecil & PinecilV2

+build-multilang:

+	@for modelml in $(MODELS_ML); do \

+		$(MAKE)  -C source/  -j2  model=$${modelml}  firmware-multi_compressed_European  firmware-multi_compressed_Bulgarian+Russian+Serbian+Ukrainian  firmware-multi_Chinese+Japanese ; \

+		mkdir  -p  $(OUT_DIR)/$${modelml}_multi-lang ; \

+		cp  $(OUT_HEX)/$${modelml}_multi_*.bin   $(OUT_DIR)/$${modelml}_multi-lang ; \

+		cp  $(OUT_HEX)/$${modelml}_multi_*.hex   $(OUT_DIR)/$${modelml}_multi-lang ; \

+		cp  $(OUT_HEX)/$${modelml}_multi_*.dfu   $(OUT_DIR)/$${modelml}_multi-lang ; \

+	done;

+	@echo "Resulting output directory: $(OUT_DIR)"

+

+# target to reproduce zips according to github CI settings; artifacts will be in $(OUT_DIR)/CI/*.zip

+ci: tests  build-all  build-multilang

+	@mkdir  -p  $(OUT_DIR)/metadata;

+	@for m in $(MODELS) $(MODELS_MULTILANG); do \

+		cp LICENSE scripts/LICENSE_RELEASE.md  $(OUT_DIR)/$${m}/ ; \

+		$(ZIP)  $(OUT_DIR)/$${m}.zip  $(OUT_DIR)/$${m} ;           \

+		./source/metadata.py  $${m}.json  $${m};                   \

+		cp  $(OUT_HEX)/$${m}.json  $(OUT_DIR)/metadata;            \

+	done;

+	@$(ZIP)  $(OUT_DIR)/metadata.zip  $(OUT_DIR)/metadata

+	@mkdir -p  $(OUT_DIR)/CI

+	@mv        $(OUT_DIR)/*.zip       $(OUT_DIR)/CI

+	@chmod  0777  $(OUT_DIR)/CI

+	@chmod  0666  $(OUT_DIR)/CI/*.zip

+	@echo "Resulting artifacts directory: $(OUT_DIR)/CI"

+

 # pass-through target for Makefile inside source/ dir

 %:

 	$(MAKE)  -C source/  $@

@@ -191,4 +229,9 @@ clean-build:
 clean-full: clean-build  docker-clean

 

 # phony targets

-.PHONY:  help  list  docker-check  docker-shell  docker-build  docker-clean  docs  docs-deploy  test-md  test-sh  test-py  test-ccpp  tests  clean-build  clean-full

+.PHONY:  help  list

+.PHONY:  docker-check  docker-shell  docker-build  docker-clean

+.PHONY:  docs  docs-deploy

+.PHONY:  test-md  test-sh  test-py  test-ccpp  tests

+.PHONY:  build-all  build-multilang  ci

+.PHONY:  clean-build  clean-full

diff --git a/scripts/IronOS.Dockerfile b/scripts/IronOS.Dockerfile
index 1543e1b0..c4514c1a 100644
--- a/scripts/IronOS.Dockerfile
+++ b/scripts/IronOS.Dockerfile
@@ -9,7 +9,7 @@ WORKDIR /build/ironos
 # Installing the two compilers (ARM & RISCV), python3 & pip, clang tools, etc.:

 ## - compilers: gcc-*, newlib-*

 ## - python3: py*, black (required to check Python code formatting)

-## - misc: findutils, make, git, diffutils

+## - misc: findutils, make, git, diffutils, zip

 ## - musl-dev (required for the multi lang firmwares)

 ## - clang (required for clang-format to check C++ code formatting)

 ## - shellcheck (to check sh scripts)

@@ -17,7 +17,7 @@ WORKDIR /build/ironos
 ARG APK_COMPS="gcc-riscv-none-elf gcc-arm-none-eabi newlib-riscv-none-elf \

                newlib-arm-none-eabi"

 ARG APK_PYTHON="python3 py3-pip black"

-ARG APK_MISC="findutils make git diffutils"

+ARG APK_MISC="findutils make git diffutils zip"

 ARG APK_DEV="musl-dev clang bash clang-extra-tools shellcheck"

 

 # PIP packages to check & test Python code

diff --git a/source/metadata.py b/source/metadata.py
index 775e2eca..9700ed59 100755
--- a/source/metadata.py
+++ b/source/metadata.py
@@ -10,27 +10,33 @@ import sys
 # Creates an index metadata json file of the hexfiles folder
 # This is used by automation like the Pinecil updater
 
-
-if len(sys.argv) != 2:
-    print("Requires the output json name as an arg")
+if len(sys.argv) < 2 or len(sys.argv) > 3:
+    print("Usage: metadata.py OUTPUT_FILE [model]")
+    print("  OUTPUT_FILE      - the name of output file in json format with meta info about binary files")
+    print("  model [optional] - name of the model (as for `make model=NAME`) to scan files for explicitly (all files in source/Hexfile by default otherwise)")
     exit(1)
 
+# If model is provided explicitly to scan related files only for json output, then process the argument
+ModelName = None
+if len(sys.argv) == 3:
+    ModelName = sys.argv[2]
+    if ModelName.endswith("_multi-lang"):
+        # rename on-the-fly for direct compatibility with make targets like PINECILMODEL_multi-lang
+        ModelName = ModelName.rstrip("-lang")
+
 HERE = Path(__file__).resolve().parent
 
 HexFileFolder = os.path.join(HERE, "Hexfile")
 OutputJSONPath = os.path.join(HexFileFolder, sys.argv[1])
 TranslationsFilesPath = os.path.join(HERE.parent, "Translations")
 
-
 def load_json(filename: str):
     with open(filename) as f:
         return json.loads(f.read())
 
-
 def read_git_tag():
     return f"{subprocess.check_output(['git', 'rev-parse', '--short=7', 'HEAD']).strip().decode('ascii').upper()}"
 
-
 def read_version():
     with open(HERE / "version.h") as version_file:
         for line in version_file:
@@ -40,10 +46,9 @@ def read_version():
                     return matches[0]
     raise Exception("Could not parse version")
 
-
 # Fetch our file listings
 translation_files = [os.path.join(TranslationsFilesPath, f) for f in os.listdir(TranslationsFilesPath) if os.path.isfile(os.path.join(TranslationsFilesPath, f)) and f.endswith(".json")]
-output_files = [os.path.join(HexFileFolder, f) for f in os.listdir(HexFileFolder) if os.path.isfile(os.path.join(HexFileFolder, f))]
+output_files = [os.path.join(HexFileFolder, f) for f in sorted(os.listdir(HexFileFolder)) if os.path.isfile(os.path.join(HexFileFolder, f))]
 
 parsed_languages = {}
 for path in translation_files:
@@ -61,6 +66,13 @@ for file_path in output_files:
     if file_path.endswith(".hex") or file_path.endswith(".dfu"):
         # Find out what language this file is
         name: str = os.path.basename(file_path)
+        if ModelName is not None:
+            # If ModelName is provided as the second argument (compatible with make model=NAME fully) but current file name doesn't match the model name, then skip it
+            if not name.startswith(ModelName + "_"):
+                continue
+            # If build of interest is not multi-lang one but scanning one is not MODEL_LANG-ID here, then skip it to avoid mess in json between MODEL_LANG-ID & MODEL_multi'
+            if not ModelName.endswith("_multi") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name):
+                continue
         matches = re.findall(r"^([a-zA-Z0-9]+)_(.+)\.(.+)$", name)
         if matches:
             matches = matches[0]