From eda3ad3dd4ab4559904c376e7df802132703fb50 Mon Sep 17 00:00:00 2001 From: Jelle Teijema Date: Fri, 22 Dec 2023 12:54:54 +0100 Subject: [PATCH] Change multiple models to Multimodel (#51) * rename multiple models to Multimodel * rename MultiModel example folder * Add old multiple_models key * Add compatibility for old multiple_models keyword --- .github/workflows/ci-workflow.yml | 8 ++++---- README.md | 8 ++++---- asreviewcontrib/makita/entrypoint.py | 18 +++++++++++------- ...ltiple_models.py => template_multimodel.py} | 6 +++--- ...mplate => template_multimodel.txt.template} | 4 ++-- examples/README.md | 6 +++--- .../README.md | 2 +- .../data/Smid_2020.csv | 0 .../data/van_de_Schoot_2018.csv | 0 .../jobs.sh | 0 .../scripts/get_plot.py | 0 .../scripts/merge_descriptives.py | 0 .../scripts/merge_metrics.py | 0 .../scripts/merge_tds.py | 0 14 files changed, 28 insertions(+), 24 deletions(-) rename asreviewcontrib/makita/{template_multiple_models.py => template_multimodel.py} (96%) rename asreviewcontrib/makita/templates/{template_multiple_models.txt.template => template_multimodel.txt.template} (98%) rename examples/{multiple_models_example => multimodel_example}/README.md (94%) rename examples/{multiple_models_example => multimodel_example}/data/Smid_2020.csv (100%) rename examples/{multiple_models_example => multimodel_example}/data/van_de_Schoot_2018.csv (100%) rename examples/{multiple_models_example => multimodel_example}/jobs.sh (100%) rename examples/{multiple_models_example => multimodel_example}/scripts/get_plot.py (100%) rename examples/{multiple_models_example => multimodel_example}/scripts/merge_descriptives.py (100%) rename examples/{multiple_models_example => multimodel_example}/scripts/merge_metrics.py (100%) rename examples/{multiple_models_example => multimodel_example}/scripts/merge_tds.py (100%) diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 8e5668eb..5c257afa 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -24,10 +24,10 @@ jobs: cd tmp mkdir -p basic/data mkdir -p arfi/data - mkdir -p multiple_models/data + mkdir -p multimodel/data cp ../.github/workflows/test_data/labels.csv basic/data/labels.csv cp ../.github/workflows/test_data/labels.csv arfi/data/labels.csv - cp ../.github/workflows/test_data/labels.csv multiple_models/data/labels.csv + cp ../.github/workflows/test_data/labels.csv multimodel/data/labels.csv - name: Test makita templates run: | cd tmp/basic @@ -36,8 +36,8 @@ jobs: cd ../arfi asreview makita template arfi | tee output.txt grep -q "ERROR" output.txt && exit 1 || true - cd ../multiple_models - asreview makita template multiple_models | tee output.txt + cd ../multimodel + asreview makita template multimodel | tee output.txt grep -q "ERROR" output.txt && exit 1 || true - name: Run ShellCheck if: ${{ matrix.os != 'windows-latest' }} diff --git a/README.md b/README.md index c81edb76..7fd234d8 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,9 @@ optional arguments: --stop_if STOP_IF The number of label actions to simulate. Default 'min' will stop simulating when all relevant records are found. ``` -### Multiple models template +### Multimodel template -command: `multiple_models` +command: `multimodel` The multiple model template prepares a script for running a simulation study comparing multiple models for one dataset and a fixed set of priors (one relevant and one irrelevant record; identical across models). @@ -191,7 +191,7 @@ want to exclude the combinations of `nb` with `doc2vec` and `logistic` with `tfidf`, use the following command: ```console -asreview makita template multiple_models --classifiers logistic nb --feature_extractors tfidf doc2vec --impossible_models nb,doc2vec logistic,tfidf +asreview makita template multimodel --classifiers logistic nb --feature_extractors tfidf doc2vec --impossible_models nb,doc2vec logistic,tfidf ``` ## Advanced usage @@ -250,7 +250,7 @@ The following scripts are available: #### Time to Discovery Tables -The 'merge_tds.py' script creates a table of the time to discovery (TD) values for each dataset, with each row corresponding to each record ID of the relevant records in a dataset, and the columns correspond to each simulation run (e.g, for the multiple models template each column corresponds to a simualtion run with each active learning model). Additionally, the tables includes the average-record-TD values (the average of the TD values for a record across multiple simulation runs), and the average-simulation-TD values (the average of the TD values across all records for a single simulation run). +The 'merge_tds.py' script creates a table of the time to discovery (TD) values for each dataset, with each row corresponding to each record ID of the relevant records in a dataset, and the columns correspond to each simulation run (e.g, for the Multimodel template each column corresponds to a simualtion run with each active learning model). Additionally, the tables includes the average-record-TD values (the average of the TD values for a record across multiple simulation runs), and the average-simulation-TD values (the average of the TD values across all records for a single simulation run). #### Run Makita via Docker diff --git a/asreviewcontrib/makita/entrypoint.py b/asreviewcontrib/makita/entrypoint.py index a9bdb048..0838b4c0 100644 --- a/asreviewcontrib/makita/entrypoint.py +++ b/asreviewcontrib/makita/entrypoint.py @@ -8,7 +8,7 @@ from asreviewcontrib.makita.config import TEMPLATES_FP from asreviewcontrib.makita.template_arfi import render_jobs_arfi from asreviewcontrib.makita.template_basic import render_jobs_basic -from asreviewcontrib.makita.template_multiple_models import render_jobs_multiple_models +from asreviewcontrib.makita.template_multimodel import render_jobs_multimodel from asreviewcontrib.makita.utils import FileHandler @@ -88,7 +88,7 @@ def execute(self, argv): # noqa: C901 "--n_runs", type=int, default=1, - help="Number of runs. Only for templates 'basic' and 'multiple_models'. " + help="Number of runs. Only for templates 'basic' and 'multimodel'. " "Default: 1.", ) parser_template.add_argument( @@ -149,21 +149,21 @@ def execute(self, argv): # noqa: C901 "--classifiers", nargs="+", default=["logistic", "nb", "rf", "svm"], - help="Classifiers to use. Only for template 'multiple_models'. " + help="Classifiers to use. Only for template 'multimodel'. " "Default: ['logistic', 'nb', 'rf', 'svm']", ) parser_template.add_argument( "--feature_extractors", nargs="+", default=["doc2vec", "sbert", "tfidf"], - help="Feature extractors to use. Only for template 'multiple_models'. " + help="Feature extractors to use. Only for template 'multimodel'. " "Default: ['doc2vec', 'sbert', 'tfidf']", ) parser_template.add_argument( "--impossible_models", nargs="+", default=["nb,doc2vec", "nb,sbert"], - help="Model combinations to exclude. Only for template 'multiple_models'. " + help="Model combinations to exclude. Only for template 'multimodel'. " "Default: ['nb,doc2vec', 'nb,sbert']", ) @@ -194,6 +194,10 @@ def _template_cli(self, args): def _template(self, args): """Generate a template.""" + # backwards compatibility for 'multiple_models' + if args.name == "multiple_models": + args.name = "multimodel" + # check if a custom template is used, otherwise use the default template fp_template = args.template or (args.name and _get_template_fp(args.name)) _is_valid_template(fp_template) @@ -252,9 +256,9 @@ def _template(self, args): platform_sys=args.platform, ) - elif args.name in ["multiple_models"]: + elif args.name in ["multimodel"]: # render jobs - job = render_jobs_multiple_models( + job = render_jobs_multimodel( datasets, output_folder=Path(args.o), create_wordclouds=args.no_wordclouds, diff --git a/asreviewcontrib/makita/template_multiple_models.py b/asreviewcontrib/makita/template_multimodel.py similarity index 96% rename from asreviewcontrib/makita/template_multiple_models.py rename to asreviewcontrib/makita/template_multimodel.py index c86b7aa5..c702cd67 100644 --- a/asreviewcontrib/makita/template_multiple_models.py +++ b/asreviewcontrib/makita/template_multimodel.py @@ -1,4 +1,4 @@ -"""Render multiple_models template.""" +"""Render multimodel template.""" import os import platform @@ -11,7 +11,7 @@ from asreviewcontrib.makita.utils import check_filename_dataset -def render_jobs_multiple_models( +def render_jobs_multimodel( datasets, output_folder="output", n_runs=1, @@ -88,7 +88,7 @@ def render_jobs_multiple_models( "doc", datasets=datasets, template_name=template.name - if template.name == "multiple_models" + if template.name == "multimodel" else "custom", template_name_long=template.name_long, template_scripts=template.scripts, diff --git a/asreviewcontrib/makita/templates/template_multiple_models.txt.template b/asreviewcontrib/makita/templates/template_multimodel.txt.template similarity index 98% rename from asreviewcontrib/makita/templates/template_multiple_models.txt.template rename to asreviewcontrib/makita/templates/template_multimodel.txt.template index 3783fa08..83b304b7 100644 --- a/asreviewcontrib/makita/templates/template_multiple_models.txt.template +++ b/asreviewcontrib/makita/templates/template_multimodel.txt.template @@ -1,5 +1,5 @@ --- -name: multiple_models +name: multimodel name_long: Basic simulation for every possible combination of selected models scripts: @@ -12,7 +12,7 @@ docs: - README.md --- -{# This is a template for the multiple_models method #} +{# This is a template for the multimodel method #} # version {{ version }} # Create folder structure. By default, the folder 'output' is used to store output. diff --git a/examples/README.md b/examples/README.md index 5c543ad1..8f151abf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,8 +11,8 @@ synergy_dataset get -d van_de_Schoot_2018 Smid_2020 -o examples/basic_example/da cd examples/basic_example asreview makita template basic cd ../.. -synergy_dataset get -d van_de_Schoot_2018 Smid_2020 -o examples/multiple_models_example/data -l -cd examples/multiple_models_example -asreview makita template multiple_models +synergy_dataset get -d van_de_Schoot_2018 Smid_2020 -o examples/multimodel_example/data -l +cd examples/multimodel_example +asreview makita template multimodel cd ../.. ``` diff --git a/examples/multiple_models_example/README.md b/examples/multimodel_example/README.md similarity index 94% rename from examples/multiple_models_example/README.md rename to examples/multimodel_example/README.md index ef399bdf..e0bf8f57 100644 --- a/examples/multiple_models_example/README.md +++ b/examples/multimodel_example/README.md @@ -2,7 +2,7 @@ *This project was rendered with ASReview-Makita version 0.0.0.* -This project was rendered from the Makita-multiple_models template. See [asreview/asreview-makita#templates](https://github.com/asreview/asreview-makita#templates) for template rules and formats. +This project was rendered from the Makita-multimodel template. See [asreview/asreview-makita#templates](https://github.com/asreview/asreview-makita#templates) for template rules and formats. The template is described as: 'Basic simulation for every possible combination of selected models'. diff --git a/examples/multiple_models_example/data/Smid_2020.csv b/examples/multimodel_example/data/Smid_2020.csv similarity index 100% rename from examples/multiple_models_example/data/Smid_2020.csv rename to examples/multimodel_example/data/Smid_2020.csv diff --git a/examples/multiple_models_example/data/van_de_Schoot_2018.csv b/examples/multimodel_example/data/van_de_Schoot_2018.csv similarity index 100% rename from examples/multiple_models_example/data/van_de_Schoot_2018.csv rename to examples/multimodel_example/data/van_de_Schoot_2018.csv diff --git a/examples/multiple_models_example/jobs.sh b/examples/multimodel_example/jobs.sh similarity index 100% rename from examples/multiple_models_example/jobs.sh rename to examples/multimodel_example/jobs.sh diff --git a/examples/multiple_models_example/scripts/get_plot.py b/examples/multimodel_example/scripts/get_plot.py similarity index 100% rename from examples/multiple_models_example/scripts/get_plot.py rename to examples/multimodel_example/scripts/get_plot.py diff --git a/examples/multiple_models_example/scripts/merge_descriptives.py b/examples/multimodel_example/scripts/merge_descriptives.py similarity index 100% rename from examples/multiple_models_example/scripts/merge_descriptives.py rename to examples/multimodel_example/scripts/merge_descriptives.py diff --git a/examples/multiple_models_example/scripts/merge_metrics.py b/examples/multimodel_example/scripts/merge_metrics.py similarity index 100% rename from examples/multiple_models_example/scripts/merge_metrics.py rename to examples/multimodel_example/scripts/merge_metrics.py diff --git a/examples/multiple_models_example/scripts/merge_tds.py b/examples/multimodel_example/scripts/merge_tds.py similarity index 100% rename from examples/multiple_models_example/scripts/merge_tds.py rename to examples/multimodel_example/scripts/merge_tds.py