From 0a9b122a0d607763992798cfe5d8a115be0f79c4 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Tue, 5 Dec 2023 18:27:46 +0100 Subject: [PATCH 01/18] adds a --server option to pack --- .omero | 2 +- README.md | 5 + src/omero_cli_transfer.py | 394 +++++++++++++++++++----------- test/integration/test_transfer.py | 23 ++ 4 files changed, 277 insertions(+), 147 deletions(-) diff --git a/.omero b/.omero index 421c3ae..3824e85 160000 --- a/.omero +++ b/.omero @@ -1 +1 @@ -Subproject commit 421c3aeb9861fcdce1f29f815c425f877de50159 +Subproject commit 3824e8568d3eb1352eca3a413811b7d665fb3aef diff --git a/README.md b/README.md index 929b60d..72f6536 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,10 @@ about the files (name, mimetype). `--plugin` allows you to export omero data to a desired format by using an external plugin. See for example the [arc plugin](https://github.com/cmohl2013/omero-arc), which exports omero projects to ARC repositories. +`--server` creates the transfer.xml file but does not copy data +or generate an archive. The last cli argument is the path where the `transfer.xml` +file will be written + Examples: ``` @@ -71,6 +75,7 @@ omero transfer pack --zip Image:123 transfer_pack.zip omero transfer pack Dataset:1111 /home/user/new_folder/new_pack.tar omero transfer pack 999 tarfile.tar # equivalent to Project:999 omero transfer pack --plugin arc Project:999 path/to/my/arc/repo +omero transfer pack --server Dataset:1111 /home/user/new_folder ``` ## `omero transfer unpack` diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index fa242d9..d375fee 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -40,13 +40,13 @@ MD5_BUF_SIZE = 65536 -HELP = ("""Transfer objects and annotations between servers. +HELP = """Transfer objects and annotations between servers. Both subcommands (pack and unpack) will use an existing OMERO session created via CLI or prompt the user for parameters to create one. -""") +""" -PACK_HELP = ("""Creates transfer packet for moving objects. +PACK_HELP = """Creates transfer packet for moving objects. This subcommand creates a transfer packet for moving objects between OMERO server instances. @@ -85,15 +85,20 @@ orig_group`), other options are `none`, `img_id`, `timestamp`, `software`, `version`, `md5`, `hostname`, `db_id`, `orig_user`, `orig_group`. +--server creates the transfer.xml file but does not copy data +or generate an archive. The last cli argument is the path where the `transfer.xml` +file will be written + Examples: omero transfer pack Image:123 transfer_pack.tar omero transfer pack --zip Image:123 transfer_pack.zip omero transfer pack Dataset:1111 /home/user/new_folder/new_pack.tar omero transfer pack 999 tarfile.tar # equivalent to Project:999 omero transfer pack 1 transfer_pack.tar --metadata img_id version db_id -""") +omero transfer pack --server Dataset:1111 /home/user/new_folder +""" -UNPACK_HELP = ("""Unpacks a transfer packet into an OMERO hierarchy. +UNPACK_HELP = """Unpacks a transfer packet into an OMERO hierarchy. Unpacks an existing transfer packet, imports images as orphans and uses the XML contained in the transfer packet to re-create @@ -132,9 +137,9 @@ omero transfer unpack --output /home/user/optional_folder --ln_s omero transfer unpack --folder /home/user/unpacked_folder/ --skip upgrade omero transfer unpack pack.tar --metadata db_id orig_user hostname -""") +""" -PREPARE_HELP = ("""Creates an XML from a folder with images. +PREPARE_HELP = """Creates an XML from a folder with images. Creates an XML file appropriate for usage with `omero transfer unpack` from a folder that contains image files, rather than a source OMERO server. This @@ -156,7 +161,7 @@ Examples: omero transfer prepare /home/user/folder_with_files omero transfer prepare --filelist /home/user/file_with_paths.txt -""") +""" def gateway_required(func: Callable) -> Callable: @@ -165,13 +170,14 @@ def gateway_required(func: Callable) -> Callable: a BlitzGateway (self.gateway), and makes sure that all services of the Blitzgateway are closed again. """ + @wraps(func) def _wrapper(self, *args, **kwargs): self.client = self.ctx.conn(*args) self.session = self.client.getSessionId() self.gateway = BlitzGateway(client_obj=self.client) router = self.client.getRouter(self.client.getCommunicator()) - self.hostname = str(router).split('-h ')[-1].split()[0] + self.hostname = str(router).split("-h ")[-1].split()[0] try: return func(self, *args, **kwargs) finally: @@ -179,11 +185,11 @@ def _wrapper(self, *args, **kwargs): self.gateway.close(hard=False) self.gateway = None self.client = None + return _wrapper class TransferControl(GraphControl): - def _configure(self, parser): parser.add_login_arguments() sub = parser.sub() @@ -192,89 +198,133 @@ def _configure(self, parser): prepare = parser.add(sub, self.prepare, PREPARE_HELP) render_type = ProxyStringType("Project") - obj_help = ("Object to be packed for transfer") + obj_help = "Object to be packed for transfer" pack.add_argument("object", type=render_type, help=obj_help) - file_help = ("Path to where the packed file will be saved") + file_help = "Path to where the packed file will be saved" pack.add_argument( - "--zip", help="Pack into a zip file rather than a tarball", - action="store_true") + "--zip", + help="Pack into a zip file rather than a tarball", + action="store_true", + ) pack.add_argument( - "--figure", help="Include OMERO.Figures into the pack" - " (caveats apply)", - action="store_true") + "--figure", + help="Include OMERO.Figures into the pack" " (caveats apply)", + action="store_true", + ) pack.add_argument( - "--barchive", help="Pack into a file compliant with Bioimage" - " Archive submission standards", - action="store_true") + "--barchive", + help="Pack into a file compliant with Bioimage" + " Archive submission standards", + action="store_true", + ) pack.add_argument( - "--rocrate", help="Pack into a file compliant with " - "RO-Crate standards", - action="store_true") + "--rocrate", + help="Pack into a file compliant with " "RO-Crate standards", + action="store_true", + ) pack.add_argument( - "--simple", help="Pack into a human-readable package file", - action="store_true") + "--simple", + help="Pack into a human-readable package file", + action="store_true", + ) pack.add_argument( "--metadata", - choices=['all', 'none', 'img_id', 'timestamp', - 'software', 'version', 'md5', 'hostname', 'db_id', - 'orig_user', 'orig_group'], nargs='+', - help="Metadata field to be added to MapAnnotation" + choices=[ + "all", + "none", + "img_id", + "timestamp", + "software", + "version", + "md5", + "hostname", + "db_id", + "orig_user", + "orig_group", + ], + nargs="+", + help="Metadata field to be added to MapAnnotation", + ) + pack.add_argument( + "--server", + help="Only generate the xml file, don't create the archive", ) pack.add_argument( "--plugin", help="Use external plugin for packing.", type=str) pack.add_argument("filepath", type=str, help=file_help) - file_help = ("Path to where the zip file is saved") + file_help = ( + "Path to where the zip file is saved " + "(if the --server option is passed, path to the output xml)" + ) unpack.add_argument("filepath", type=str, help=file_help) unpack.add_argument( - "--ln_s_import", help="Use in-place import", - action="store_true") + "--ln_s_import", help="Use in-place import", action="store_true" + ) unpack.add_argument( - "--merge", help="Use existing entities if possible", - action="store_true") + "--merge", help="Use existing entities if possible", action="store_true" + ) unpack.add_argument( - "--figure", help="Use OMERO.Figures if present" - " (caveats apply)", - action="store_true") + "--figure", + help="Use OMERO.Figures if present" " (caveats apply)", + action="store_true", + ) unpack.add_argument( - "--folder", help="Pass path to a folder rather than a pack", - action="store_true") + "--folder", + help="Pass path to a folder rather than a pack", + action="store_true", + ) unpack.add_argument( - "--output", type=str, help="Output directory where zip " - "file will be extracted" + "--output", + type=str, + help="Output directory where zip " "file will be extracted", ) unpack.add_argument( - "--skip", choices=['all', 'checksum', 'thumbnails', 'minmax', - 'upgrade'], - help="Skip options to be passed to omero import" + "--skip", + choices=["all", "checksum", "thumbnails", "minmax", "upgrade"], + help="Skip options to be passed to omero import", ) unpack.add_argument( "--metadata", - choices=['all', 'none', 'img_id', 'plate_id', 'timestamp', - 'software', 'version', 'md5', 'hostname', 'db_id', - 'orig_user', 'orig_group'], nargs='+', - help="Metadata field to be added to MapAnnotation" + choices=[ + "all", + "none", + "img_id", + "plate_id", + "timestamp", + "software", + "version", + "md5", + "hostname", + "db_id", + "orig_user", + "orig_group", + ], + nargs="+", + help="Metadata field to be added to MapAnnotation", ) - folder_help = ("Path to folder with image files") + folder_help = "Path to folder with image files" prepare.add_argument("folder", type=str, help=folder_help) prepare.add_argument( - "--filelist", help="Pass path to a filelist rather than a folder", - action="store_true") + "--filelist", + help="Pass path to a filelist rather than a folder", + action="store_true", + ) @gateway_required def pack(self, args): - """ Implements the 'pack' command """ + """Implements the 'pack' command""" self.__pack(args) @gateway_required def unpack(self, args): - """ Implements the 'unpack' command """ + """Implements the 'unpack' command""" self.__unpack(args) @gateway_required def prepare(self, args): - """ Implements the 'prepare' command """ + """Implements the 'prepare' command""" self.__prepare(args) def _get_path_to_repo(self) -> List[str]: @@ -290,8 +340,7 @@ def _get_path_to_repo(self) -> List[str]: mrepos.append(path) return mrepos - def _copy_files(self, id_list: Dict[str, Any], folder: str, - conn: BlitzGateway): + def _copy_files(self, id_list: Dict[str, Any], folder: str, conn: BlitzGateway): if not isinstance(id_list, dict): raise TypeError("id_list must be a dict") if not all(isinstance(item, str) for item in id_list.keys()): @@ -321,10 +370,10 @@ def _copy_files(self, id_list: Dict[str, Any], folder: str, id = "File" + id if rel_path == "pixel_images": filepath = str(Path(subfolder) / (str(clean_id) + ".tiff")) - cli.invoke(['export', '--file', filepath, id]) + cli.invoke(["export", "--file", filepath, id]) downloaded_ids.append(id) else: - cli.invoke(['download', id, subfolder]) + cli.invoke(["download", id, subfolder]) if dtype == "Image": obj = conn.getObject("Image", clean_id) fileset = obj.getFileset() @@ -334,31 +383,39 @@ def _copy_files(self, id_list: Dict[str, Any], folder: str, def _package_files(self, tar_path: str, zip: bool, folder: str): if zip: print("Creating zip file...") - shutil.make_archive(tar_path, 'zip', folder) + shutil.make_archive(tar_path, "zip", folder) else: print("Creating tar file...") - shutil.make_archive(tar_path, 'tar', folder) + shutil.make_archive(tar_path, "tar", folder) def _process_metadata(self, metadata: Union[List[str], None]): if not metadata: - metadata = ['all'] + metadata = ["all"] if "all" in metadata: metadata.remove("all") - metadata.extend(["img_id", "plate_id", "timestamp", "software", - "version", "hostname", "md5", "orig_user", - "orig_group"]) + metadata.extend( + [ + "img_id", + "plate_id", + "timestamp", + "software", + "version", + "hostname", + "md5", + "orig_user", + "orig_group", + ] + ) if "none" in metadata: metadata = None if metadata: metadata = list(set(metadata)) self.metadata = metadata - def _fix_pixels_image_simple(self, ome: OME, folder: str, filepath: str - ) -> OME: + def _fix_pixels_image_simple(self, ome: OME, folder: str, filepath: str) -> OME: newome = copy.deepcopy(ome) for ann in ome.structured_annotations: - if isinstance(ann.value, str) and\ - ann.value.startswith("pixel_images"): + if isinstance(ann.value, str) and ann.value.startswith("pixel_images"): for img in newome.images: for ref in img.annotation_refs: if ref.id == ann.id: @@ -374,28 +431,41 @@ def _fix_pixels_image_simple(self, ome: OME, folder: str, filepath: str rel_path = str(Path(path2).parent) subfolder = os.path.join(str(Path(folder)), rel_path) os.makedirs(subfolder, mode=DIR_PERM, exist_ok=True) - shutil.move(os.path.join(str(Path(folder)), path1), - os.path.join(str(Path(folder)), path2)) + shutil.move( + os.path.join(str(Path(folder)), path1), + os.path.join(str(Path(folder)), path2), + ) if os.path.exists(os.path.join(str(Path(folder)), "pixel_images")): shutil.rmtree(os.path.join(str(Path(folder)), "pixel_images")) - with open(filepath, 'w') as fp: + with open(filepath, "w") as fp: print(to_xml(newome), file=fp) fp.close() return newome def __pack(self, args): - if isinstance(args.object, Image) or isinstance(args.object, Plate) \ - or isinstance(args.object, Screen): + if ( + isinstance(args.object, Image) + or isinstance(args.object, Plate) + or isinstance(args.object, Screen) + ): if args.barchive: - raise ValueError("Single image, plate or screen cannot be " - "packaged for Bioimage Archive") + raise ValueError( + "Single image, plate or screen cannot be " + "packaged for Bioimage Archive" + ) if isinstance(args.object, Plate) or isinstance(args.object, Screen): if args.rocrate: - raise ValueError("Single image, plate or screen cannot be " - "packaged in a RO-Crate") + raise ValueError( + "Single image, plate or screen cannot be " "packaged in a RO-Crate" + ) if args.simple: - raise ValueError("Single plate or screen cannot be " - "packaged in human-readable format") + raise ValueError( + "Single plate or screen cannot be " + "packaged in human-readable format" + ) + if args.server and args.simple: + raise ValueError("The --server and --simple options are incompatible") + if isinstance(args.object, Image): src_datatype, src_dataid = "Image", args.object.id elif isinstance(args.object, Dataset): @@ -411,19 +481,31 @@ def __pack(self, args): return export_types = (args.rocrate, args.barchive, args.simple) if sum(1 for ct in export_types if ct) > 1: - raise ValueError("Only one special export type (RO-Crate, Bioimage" - " Archive, human-readable) can be specified at " - "once") + raise ValueError( + "Only one special export type (RO-Crate, Bioimage" + " Archive, human-readable) can be specified at " + "once" + ) self.metadata = [] self._process_metadata(args.metadata) obj = self.gateway.getObject(src_datatype, src_dataid) if obj is None: - raise ValueError("Object not found or outside current" - " permissions for current user.") + raise ValueError( + "Object not found or outside current" " permissions for current user." + ) print("Populating xml...") - tar_path = Path(args.filepath) - folder = str(tar_path) + "_folder" + if not args.server: + tar_path = Path(args.filepath) + folder = str(tar_path) + "_folder" + else: + tar_path = Path(args.filepath) + if tar_path.suffix: + folder = os.path.splitext(tar_path)[0] + print("Output will be written to {folder}") + else: + folder = tar_path os.makedirs(folder, mode=DIR_PERM, exist_ok=True) + if args.barchive: md_fp = str(Path(folder) / "submission.tsv") elif args.rocrate: @@ -431,19 +513,26 @@ def __pack(self, args): else: md_fp = str(Path(folder) / "transfer.xml") print(f"Saving metadata at {md_fp}.") - ome, path_id_dict = populate_xml(src_datatype, src_dataid, md_fp, - self.gateway, self.hostname, - args.barchive, args.simple, - args.figure, - self.metadata) - print("Starting file copy...") - self._copy_files(path_id_dict, folder, self.gateway) + ome, path_id_dict = populate_xml( + src_datatype, + src_dataid, + md_fp, + self.gateway, + self.hostname, + args.barchive, + args.simple, + args.figure, + self.metadata, + ) + if not args.server: + print("Starting file copy...") + self._copy_files(path_id_dict, folder, self.gateway) + if args.simple: self._fix_pixels_image_simple(ome, folder, md_fp) if args.barchive: print(f"Creating Bioimage Archive TSV at {md_fp}.") - populate_tsv(src_datatype, ome, md_fp, - path_id_dict, folder) + populate_tsv(src_datatype, ome, md_fp, path_id_dict, folder) if args.rocrate: print(f"Creating RO-Crate metadata at {md_fp}.") populate_rocrate(src_datatype, ome, os.path.splitext(tar_path)[0], @@ -477,11 +566,12 @@ def __pack(self, args): tmp_path=Path(folder), image_filenames_mapping=path_id_dict, conn=self.gateway) - else: - self._package_files(os.path.splitext(tar_path)[0], args.zip, - folder) - print("Cleaning up...") - shutil.rmtree(folder) + elif not args.server: + self._package_files(os.path.splitext(tar_path)[0], args.zip, folder) + + if not args.server: + print("Cleaning up...") + shutil.rmtree(folder) return def __unpack(self, args): @@ -489,8 +579,7 @@ def __unpack(self, args): self._process_metadata(args.metadata) if not args.folder: print(f"Unzipping {args.filepath}...") - hash, ome, folder = self._load_from_pack(args.filepath, - args.output) + hash, ome, folder = self._load_from_pack(args.filepath, args.output) else: folder = Path(args.filepath) ome = from_xml(folder / "transfer.xml") @@ -502,18 +591,28 @@ def __unpack(self, args): ln_s = True else: ln_s = False - dest_img_map = self._import_files(folder, filelist, - ln_s, args.skip, self.gateway) + dest_img_map = self._import_files( + folder, filelist, ln_s, args.skip, self.gateway + ) self._delete_all_rois(dest_img_map, self.gateway) print("Matching source and destination images...") img_map = self._make_image_map(src_img_map, dest_img_map, self.gateway) print("Creating and linking OMERO objects...") - populate_omero(ome, img_map, self.gateway, - hash, folder, self.metadata, args.merge, args.figure) + populate_omero( + ome, + img_map, + self.gateway, + hash, + folder, + self.metadata, + args.merge, + args.figure, + ) return - def _load_from_pack(self, filepath: str, output: Optional[str] = None - ) -> Tuple[str, OME, Path]: + def _load_from_pack( + self, filepath: str, output: Optional[str] = None + ) -> Tuple[str, OME, Path]: if (not filepath) or (not isinstance(filepath, str)): raise TypeError("filepath must be a string") if output and not isinstance(output, str): @@ -525,7 +624,7 @@ def _load_from_pack(self, filepath: str, output: Optional[str] = None else: folder = parent_folder / filename if Path(filepath).exists(): - with open(filepath, 'rb') as file: + with open(filepath, "rb") as file: md5 = hashlib.md5() while True: data = file.read(MD5_BUF_SIZE) @@ -533,11 +632,11 @@ def _load_from_pack(self, filepath: str, output: Optional[str] = None break md5.update(data) hash = md5.hexdigest() - if Path(filepath).suffix == '.zip': - with ZipFile(filepath, 'r') as zipobj: + if Path(filepath).suffix == ".zip": + with ZipFile(filepath, "r") as zipobj: zipobj.extractall(str(folder)) - elif Path(filepath).suffix == '.tar': - shutil.unpack_archive(filepath, str(folder), 'tar') + elif Path(filepath).suffix == ".tar": + shutil.unpack_archive(filepath, str(folder), "tar") else: raise ValueError("File is not a zip or tar file") else: @@ -545,8 +644,7 @@ def _load_from_pack(self, filepath: str, output: Optional[str] = None ome = from_xml(folder / "transfer.xml") return hash, ome, folder - def _create_image_map(self, ome: OME - ) -> Tuple[OME, DefaultDict, List[str]]: + def _create_image_map(self, ome: OME) -> Tuple[OME, DefaultDict, List[str]]: if not (type(ome) is OME): raise TypeError("XML is not valid OME format") img_map = DefaultDict(list) @@ -554,14 +652,15 @@ def _create_image_map(self, ome: OME newome = copy.deepcopy(ome) map_ref_ids = [] for ann in ome.structured_annotations: - if int(ann.id.split(":")[-1]) < 0 \ - and isinstance(ann, CommentAnnotation) \ - and ann.namespace: + if ( + int(ann.id.split(":")[-1]) < 0 + and isinstance(ann, CommentAnnotation) + and ann.namespace + ): if ann.namespace.split(":")[0] == "Image": map_ref_ids.append(ann.id) - img_map[ann.value].append(int( - ann.namespace.split(":")[-1])) - if ann.value.endswith('mock_folder'): + img_map[ann.value].append(int(ann.namespace.split(":")[-1])) + if ann.value.endswith("mock_folder"): filelist.append(ann.value.rstrip("mock_folder")) else: filelist.append(ann.value) @@ -571,23 +670,28 @@ def _create_image_map(self, ome: OME if ref.id in map_ref_ids: i.annotation_refs.remove(ref) filelist = list(set(filelist)) - img_map = DefaultDict(list, {x: sorted(img_map[x]) - for x in img_map.keys()}) + img_map = DefaultDict(list, {x: sorted(img_map[x]) for x in img_map.keys()}) return newome, img_map, filelist - def _import_files(self, folder: Path, filelist: List[str], ln_s: bool, - skip: str, gateway: BlitzGateway) -> dict: + def _import_files( + self, + folder: Path, + filelist: List[str], + ln_s: bool, + skip: str, + gateway: BlitzGateway, + ) -> dict: cli = CLI() cli.loadplugins() dest_map = {} - curr_folder = str(Path('.').resolve()) + curr_folder = str(Path(".").resolve()) for filepath in filelist: - dest_path = str(os.path.join(curr_folder, folder, '.', filepath)) - command = ['import', dest_path] + dest_path = str(os.path.join(curr_folder, folder, ".", filepath)) + command = ["import", dest_path] if ln_s: - command.append('--transfer=ln_s') + command.append("--transfer=ln_s") if skip: - command.extend(['--skip', skip]) + command.extend(["--skip", skip]) cli.invoke(command) img_ids = self._get_image_ids(dest_path, gateway) dest_map[dest_path] = img_ids @@ -614,16 +718,16 @@ def _get_image_ids(self, file_path: str, conn: BlitzGateway) -> List[str]: """ q = conn.getQueryService() params = Parameters() - path_query = str(file_path).strip('/') - params.map = {"cpath": rstring('%s%%' % path_query)} + path_query = str(file_path).strip("/") + params.map = {"cpath": rstring("%s%%" % path_query)} results = q.projection( "SELECT i.id FROM Image i" " JOIN i.fileset fs" " JOIN fs.usedFiles u" " WHERE u.clientPath LIKE :cpath", params, - conn.SERVICE_OPTS - ) + conn.SERVICE_OPTS, + ) all_image_ids = list(set(sorted([r[0].val for r in results]))) image_ids = [] for img_id in all_image_ids: @@ -634,15 +738,15 @@ def _get_image_ids(self, file_path: str, conn: BlitzGateway) -> List[str]: is_annotated = False for ann in anns: ann_content = conn.getObject("MapAnnotation", ann) - if ann_content.getNs() == \ - 'openmicroscopy.org/cli/transfer': + if ann_content.getNs() == "openmicroscopy.org/cli/transfer": is_annotated = True if not is_annotated: image_ids.append(img_id) return image_ids - def _make_image_map(self, source_map: dict, dest_map: dict, - conn: Optional[BlitzGateway] = None) -> dict: + def _make_image_map( + self, source_map: dict, dest_map: dict, conn: Optional[BlitzGateway] = None + ) -> dict: # using both source and destination file-to-image-id maps, # map image IDs between source and destination src_dict = DefaultDict(list) @@ -657,10 +761,10 @@ def _make_image_map(self, source_map: dict, dest_map: dict, for k, v in dest_map.items(): newkey = k.split("/./")[-1] dest_dict[newkey].extend(v) - src_dict = DefaultDict(list, {x: sorted(src_dict[x]) - for x in src_dict.keys()}) - dest_dict = DefaultDict(list, {x: sorted(dest_dict[x]) - for x in dest_dict.keys()}) + src_dict = DefaultDict(list, {x: sorted(src_dict[x]) for x in src_dict.keys()}) + dest_dict = DefaultDict( + list, {x: sorted(dest_dict[x]) for x in dest_dict.keys()} + ) for src_k in src_dict.keys(): src_v = src_dict[src_k] if src_k in dest_dict.keys(): @@ -674,8 +778,7 @@ def _make_image_map(self, source_map: dict, dest_map: dict, anns = 0 for j in img_obj.listAnnotations(): ns = j.getNs() - if ns.startswith( - "openmicroscopy.org/cli/transfer"): + if ns.startswith("openmicroscopy.org/cli/transfer"): anns += 1 if not anns: clean_dest.append(i) @@ -686,8 +789,7 @@ def _make_image_map(self, source_map: dict, dest_map: dict, return imgmap def __prepare(self, args): - populate_xml_folder(args.folder, args.filelist, self.gateway, - self.session) + populate_xml_folder(args.folder, args.filelist, self.gateway, self.session) return diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index 613a0bc..236c194 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -188,6 +188,29 @@ def test_pack_special(self, target_name, tmpdir): assert len(f.getmembers()) == 6 self.delete_all() + + @pytest.mark.parametrize('target_name', sorted(SUPPORTED)) + def test_pack_server(self, target_name, tmpdir): + if target_name == "datasetid" or target_name == "projectid" or\ + target_name == "idonly" or target_name == "imageid": + self.create_image(target_name=target_name) + elif target_name == "plateid" or target_name == "screenid": + self.create_plate(target_name=target_name) + target = getattr(self, target_name) + args = self.args + ["pack", target, "--server", + str(tmpdir)] + self.cli.invoke(args, strict=True) + assert os.path.exists(str(tmpdir / 'transfer.xml')) + assert os.path.getsize(str(tmpdir / 'transfer.xml')) > 0 + + args = self.args + ["pack", target, "--server", "--simple", + str(tmpdir)] + with pytest.raises(ValueError): + self.cli.invoke(args, strict=True) + self.delete_all() + + + @pytest.mark.parametrize('folder_name', TEST_FOLDERS) def test_unpack_folder(self, folder_name): self.args += ["unpack", "--folder", folder_name] From 5ddae9f7980962f28bba7dad85f93900fa9451dd Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 6 Dec 2023 09:05:11 +0100 Subject: [PATCH 02/18] removed black autoformatting --- src/omero_cli_transfer.py | 387 +++++++++++++++----------------------- 1 file changed, 151 insertions(+), 236 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index d375fee..b978cc1 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -40,13 +40,13 @@ MD5_BUF_SIZE = 65536 -HELP = """Transfer objects and annotations between servers. +HELP = ("""Transfer objects and annotations between servers. Both subcommands (pack and unpack) will use an existing OMERO session created via CLI or prompt the user for parameters to create one. -""" +""") -PACK_HELP = """Creates transfer packet for moving objects. +PACK_HELP = ("""Creates transfer packet for moving objects. This subcommand creates a transfer packet for moving objects between OMERO server instances. @@ -85,20 +85,15 @@ orig_group`), other options are `none`, `img_id`, `timestamp`, `software`, `version`, `md5`, `hostname`, `db_id`, `orig_user`, `orig_group`. ---server creates the transfer.xml file but does not copy data -or generate an archive. The last cli argument is the path where the `transfer.xml` -file will be written - Examples: omero transfer pack Image:123 transfer_pack.tar omero transfer pack --zip Image:123 transfer_pack.zip omero transfer pack Dataset:1111 /home/user/new_folder/new_pack.tar omero transfer pack 999 tarfile.tar # equivalent to Project:999 omero transfer pack 1 transfer_pack.tar --metadata img_id version db_id -omero transfer pack --server Dataset:1111 /home/user/new_folder -""" +""") -UNPACK_HELP = """Unpacks a transfer packet into an OMERO hierarchy. +UNPACK_HELP = ("""Unpacks a transfer packet into an OMERO hierarchy. Unpacks an existing transfer packet, imports images as orphans and uses the XML contained in the transfer packet to re-create @@ -137,9 +132,9 @@ omero transfer unpack --output /home/user/optional_folder --ln_s omero transfer unpack --folder /home/user/unpacked_folder/ --skip upgrade omero transfer unpack pack.tar --metadata db_id orig_user hostname -""" +""") -PREPARE_HELP = """Creates an XML from a folder with images. +PREPARE_HELP = ("""Creates an XML from a folder with images. Creates an XML file appropriate for usage with `omero transfer unpack` from a folder that contains image files, rather than a source OMERO server. This @@ -161,7 +156,7 @@ Examples: omero transfer prepare /home/user/folder_with_files omero transfer prepare --filelist /home/user/file_with_paths.txt -""" +""") def gateway_required(func: Callable) -> Callable: @@ -170,14 +165,13 @@ def gateway_required(func: Callable) -> Callable: a BlitzGateway (self.gateway), and makes sure that all services of the Blitzgateway are closed again. """ - @wraps(func) def _wrapper(self, *args, **kwargs): self.client = self.ctx.conn(*args) self.session = self.client.getSessionId() self.gateway = BlitzGateway(client_obj=self.client) router = self.client.getRouter(self.client.getCommunicator()) - self.hostname = str(router).split("-h ")[-1].split()[0] + self.hostname = str(router).split('-h ')[-1].split()[0] try: return func(self, *args, **kwargs) finally: @@ -185,11 +179,11 @@ def _wrapper(self, *args, **kwargs): self.gateway.close(hard=False) self.gateway = None self.client = None - return _wrapper class TransferControl(GraphControl): + def _configure(self, parser): parser.add_login_arguments() sub = parser.sub() @@ -198,133 +192,93 @@ def _configure(self, parser): prepare = parser.add(sub, self.prepare, PREPARE_HELP) render_type = ProxyStringType("Project") - obj_help = "Object to be packed for transfer" + obj_help = ("Object to be packed for transfer") pack.add_argument("object", type=render_type, help=obj_help) - file_help = "Path to where the packed file will be saved" + file_help = ("Path to where the packed file will be saved") pack.add_argument( - "--zip", - help="Pack into a zip file rather than a tarball", - action="store_true", - ) + "--zip", help="Pack into a zip file rather than a tarball", + action="store_true") pack.add_argument( - "--figure", - help="Include OMERO.Figures into the pack" " (caveats apply)", - action="store_true", - ) + "--figure", help="Include OMERO.Figures into the pack" + " (caveats apply)", + action="store_true") pack.add_argument( - "--barchive", - help="Pack into a file compliant with Bioimage" - " Archive submission standards", - action="store_true", - ) + "--barchive", help="Pack into a file compliant with Bioimage" + " Archive submission standards", + action="store_true") pack.add_argument( - "--rocrate", - help="Pack into a file compliant with " "RO-Crate standards", - action="store_true", - ) + "--rocrate", help="Pack into a file compliant with " + "RO-Crate standards", + action="store_true") pack.add_argument( - "--simple", - help="Pack into a human-readable package file", - action="store_true", - ) + "--simple", help="Pack into a human-readable package file", + action="store_true") pack.add_argument( "--metadata", - choices=[ - "all", - "none", - "img_id", - "timestamp", - "software", - "version", - "md5", - "hostname", - "db_id", - "orig_user", - "orig_group", - ], - nargs="+", - help="Metadata field to be added to MapAnnotation", - ) - pack.add_argument( - "--server", - help="Only generate the xml file, don't create the archive", + choices=['all', 'none', 'img_id', 'timestamp', + 'software', 'version', 'md5', 'hostname', 'db_id', + 'orig_user', 'orig_group'], nargs='+', + help="Metadata field to be added to MapAnnotation" ) pack.add_argument( "--plugin", help="Use external plugin for packing.", type=str) pack.add_argument("filepath", type=str, help=file_help) - - file_help = ( - "Path to where the zip file is saved " - "(if the --server option is passed, path to the output xml)" + pack.add_argument( + "--server", + help="Only generate the xml file, don't create the archive" ) + + file_help = ("Path to where the zip file is saved") unpack.add_argument("filepath", type=str, help=file_help) unpack.add_argument( - "--ln_s_import", help="Use in-place import", action="store_true" - ) + "--ln_s_import", help="Use in-place import", + action="store_true") unpack.add_argument( - "--merge", help="Use existing entities if possible", action="store_true" - ) + "--merge", help="Use existing entities if possible", + action="store_true") unpack.add_argument( - "--figure", - help="Use OMERO.Figures if present" " (caveats apply)", - action="store_true", - ) + "--figure", help="Use OMERO.Figures if present" + " (caveats apply)", + action="store_true") unpack.add_argument( - "--folder", - help="Pass path to a folder rather than a pack", - action="store_true", - ) + "--folder", help="Pass path to a folder rather than a pack", + action="store_true") unpack.add_argument( - "--output", - type=str, - help="Output directory where zip " "file will be extracted", + "--output", type=str, help="Output directory where zip " + "file will be extracted" ) unpack.add_argument( - "--skip", - choices=["all", "checksum", "thumbnails", "minmax", "upgrade"], - help="Skip options to be passed to omero import", + "--skip", choices=['all', 'checksum', 'thumbnails', 'minmax', + 'upgrade'], + help="Skip options to be passed to omero import" ) unpack.add_argument( "--metadata", - choices=[ - "all", - "none", - "img_id", - "plate_id", - "timestamp", - "software", - "version", - "md5", - "hostname", - "db_id", - "orig_user", - "orig_group", - ], - nargs="+", - help="Metadata field to be added to MapAnnotation", + choices=['all', 'none', 'img_id', 'plate_id', 'timestamp', + 'software', 'version', 'md5', 'hostname', 'db_id', + 'orig_user', 'orig_group'], nargs='+', + help="Metadata field to be added to MapAnnotation" ) - folder_help = "Path to folder with image files" + folder_help = ("Path to folder with image files") prepare.add_argument("folder", type=str, help=folder_help) prepare.add_argument( - "--filelist", - help="Pass path to a filelist rather than a folder", - action="store_true", - ) + "--filelist", help="Pass path to a filelist rather than a folder", + action="store_true") @gateway_required def pack(self, args): - """Implements the 'pack' command""" + """ Implements the 'pack' command """ self.__pack(args) @gateway_required def unpack(self, args): - """Implements the 'unpack' command""" + """ Implements the 'unpack' command """ self.__unpack(args) @gateway_required def prepare(self, args): - """Implements the 'prepare' command""" + """ Implements the 'prepare' command """ self.__prepare(args) def _get_path_to_repo(self) -> List[str]: @@ -340,7 +294,8 @@ def _get_path_to_repo(self) -> List[str]: mrepos.append(path) return mrepos - def _copy_files(self, id_list: Dict[str, Any], folder: str, conn: BlitzGateway): + def _copy_files(self, id_list: Dict[str, Any], folder: str, + conn: BlitzGateway): if not isinstance(id_list, dict): raise TypeError("id_list must be a dict") if not all(isinstance(item, str) for item in id_list.keys()): @@ -370,10 +325,10 @@ def _copy_files(self, id_list: Dict[str, Any], folder: str, conn: BlitzGateway): id = "File" + id if rel_path == "pixel_images": filepath = str(Path(subfolder) / (str(clean_id) + ".tiff")) - cli.invoke(["export", "--file", filepath, id]) + cli.invoke(['export', '--file', filepath, id]) downloaded_ids.append(id) else: - cli.invoke(["download", id, subfolder]) + cli.invoke(['download', id, subfolder]) if dtype == "Image": obj = conn.getObject("Image", clean_id) fileset = obj.getFileset() @@ -383,39 +338,31 @@ def _copy_files(self, id_list: Dict[str, Any], folder: str, conn: BlitzGateway): def _package_files(self, tar_path: str, zip: bool, folder: str): if zip: print("Creating zip file...") - shutil.make_archive(tar_path, "zip", folder) + shutil.make_archive(tar_path, 'zip', folder) else: print("Creating tar file...") - shutil.make_archive(tar_path, "tar", folder) + shutil.make_archive(tar_path, 'tar', folder) def _process_metadata(self, metadata: Union[List[str], None]): if not metadata: - metadata = ["all"] + metadata = ['all'] if "all" in metadata: metadata.remove("all") - metadata.extend( - [ - "img_id", - "plate_id", - "timestamp", - "software", - "version", - "hostname", - "md5", - "orig_user", - "orig_group", - ] - ) + metadata.extend(["img_id", "plate_id", "timestamp", "software", + "version", "hostname", "md5", "orig_user", + "orig_group"]) if "none" in metadata: metadata = None if metadata: metadata = list(set(metadata)) self.metadata = metadata - def _fix_pixels_image_simple(self, ome: OME, folder: str, filepath: str) -> OME: + def _fix_pixels_image_simple(self, ome: OME, folder: str, filepath: str + ) -> OME: newome = copy.deepcopy(ome) for ann in ome.structured_annotations: - if isinstance(ann.value, str) and ann.value.startswith("pixel_images"): + if isinstance(ann.value, str) and\ + ann.value.startswith("pixel_images"): for img in newome.images: for ref in img.annotation_refs: if ref.id == ann.id: @@ -431,40 +378,32 @@ def _fix_pixels_image_simple(self, ome: OME, folder: str, filepath: str) -> OME: rel_path = str(Path(path2).parent) subfolder = os.path.join(str(Path(folder)), rel_path) os.makedirs(subfolder, mode=DIR_PERM, exist_ok=True) - shutil.move( - os.path.join(str(Path(folder)), path1), - os.path.join(str(Path(folder)), path2), - ) + shutil.move(os.path.join(str(Path(folder)), path1), + os.path.join(str(Path(folder)), path2)) if os.path.exists(os.path.join(str(Path(folder)), "pixel_images")): shutil.rmtree(os.path.join(str(Path(folder)), "pixel_images")) - with open(filepath, "w") as fp: + with open(filepath, 'w') as fp: print(to_xml(newome), file=fp) fp.close() return newome def __pack(self, args): - if ( - isinstance(args.object, Image) - or isinstance(args.object, Plate) - or isinstance(args.object, Screen) - ): + if isinstance(args.object, Image) or isinstance(args.object, Plate) \ + or isinstance(args.object, Screen): if args.barchive: - raise ValueError( - "Single image, plate or screen cannot be " - "packaged for Bioimage Archive" - ) + raise ValueError("Single image, plate or screen cannot be " + "packaged for Bioimage Archive") if isinstance(args.object, Plate) or isinstance(args.object, Screen): if args.rocrate: - raise ValueError( - "Single image, plate or screen cannot be " "packaged in a RO-Crate" - ) + raise ValueError("Single image, plate or screen cannot be " + "packaged in a RO-Crate") if args.simple: - raise ValueError( - "Single plate or screen cannot be " - "packaged in human-readable format" - ) + raise ValueError("Single plate or screen cannot be " + "packaged in human-readable format") + if args.server and args.simple: - raise ValueError("The --server and --simple options are incompatible") + raise ValueError("The --server and --simple options are " + "incompatible") if isinstance(args.object, Image): src_datatype, src_dataid = "Image", args.object.id @@ -481,31 +420,24 @@ def __pack(self, args): return export_types = (args.rocrate, args.barchive, args.simple) if sum(1 for ct in export_types if ct) > 1: - raise ValueError( - "Only one special export type (RO-Crate, Bioimage" - " Archive, human-readable) can be specified at " - "once" - ) + raise ValueError("Only one special export type (RO-Crate, Bioimage" + " Archive, human-readable) can be specified at " + "once") self.metadata = [] self._process_metadata(args.metadata) obj = self.gateway.getObject(src_datatype, src_dataid) if obj is None: - raise ValueError( - "Object not found or outside current" " permissions for current user." - ) + raise ValueError("Object not found or outside current" + " permissions for current user.") print("Populating xml...") + tar_path = Path(args.filepath) if not args.server: - tar_path = Path(args.filepath) folder = str(tar_path) + "_folder" else: - tar_path = Path(args.filepath) - if tar_path.suffix: - folder = os.path.splitext(tar_path)[0] - print("Output will be written to {folder}") - else: - folder = tar_path - os.makedirs(folder, mode=DIR_PERM, exist_ok=True) + folder = os.path.splitext(tar_path)[0] + print("Output will be written to {folder}") + os.makedirs(folder, mode=DIR_PERM, exist_ok=True) if args.barchive: md_fp = str(Path(folder) / "submission.tsv") elif args.rocrate: @@ -513,17 +445,12 @@ def __pack(self, args): else: md_fp = str(Path(folder) / "transfer.xml") print(f"Saving metadata at {md_fp}.") - ome, path_id_dict = populate_xml( - src_datatype, - src_dataid, - md_fp, - self.gateway, - self.hostname, - args.barchive, - args.simple, - args.figure, - self.metadata, - ) + ome, path_id_dict = populate_xml(src_datatype, src_dataid, md_fp, + self.gateway, self.hostname, + args.barchive, args.simple, + args.figure, + self.metadata) + if not args.server: print("Starting file copy...") self._copy_files(path_id_dict, folder, self.gateway) @@ -532,7 +459,8 @@ def __pack(self, args): self._fix_pixels_image_simple(ome, folder, md_fp) if args.barchive: print(f"Creating Bioimage Archive TSV at {md_fp}.") - populate_tsv(src_datatype, ome, md_fp, path_id_dict, folder) + populate_tsv(src_datatype, ome, md_fp, + path_id_dict, folder) if args.rocrate: print(f"Creating RO-Crate metadata at {md_fp}.") populate_rocrate(src_datatype, ome, os.path.splitext(tar_path)[0], @@ -567,9 +495,8 @@ def __pack(self, args): image_filenames_mapping=path_id_dict, conn=self.gateway) elif not args.server: - self._package_files(os.path.splitext(tar_path)[0], args.zip, folder) - - if not args.server: + self._package_files(os.path.splitext(tar_path)[0], args.zip, + folder) print("Cleaning up...") shutil.rmtree(folder) return @@ -579,7 +506,8 @@ def __unpack(self, args): self._process_metadata(args.metadata) if not args.folder: print(f"Unzipping {args.filepath}...") - hash, ome, folder = self._load_from_pack(args.filepath, args.output) + hash, ome, folder = self._load_from_pack(args.filepath, + args.output) else: folder = Path(args.filepath) ome = from_xml(folder / "transfer.xml") @@ -591,28 +519,18 @@ def __unpack(self, args): ln_s = True else: ln_s = False - dest_img_map = self._import_files( - folder, filelist, ln_s, args.skip, self.gateway - ) + dest_img_map = self._import_files(folder, filelist, + ln_s, args.skip, self.gateway) self._delete_all_rois(dest_img_map, self.gateway) print("Matching source and destination images...") img_map = self._make_image_map(src_img_map, dest_img_map, self.gateway) print("Creating and linking OMERO objects...") - populate_omero( - ome, - img_map, - self.gateway, - hash, - folder, - self.metadata, - args.merge, - args.figure, - ) + populate_omero(ome, img_map, self.gateway, + hash, folder, self.metadata, args.merge, args.figure) return - def _load_from_pack( - self, filepath: str, output: Optional[str] = None - ) -> Tuple[str, OME, Path]: + def _load_from_pack(self, filepath: str, output: Optional[str] = None + ) -> Tuple[str, OME, Path]: if (not filepath) or (not isinstance(filepath, str)): raise TypeError("filepath must be a string") if output and not isinstance(output, str): @@ -624,7 +542,7 @@ def _load_from_pack( else: folder = parent_folder / filename if Path(filepath).exists(): - with open(filepath, "rb") as file: + with open(filepath, 'rb') as file: md5 = hashlib.md5() while True: data = file.read(MD5_BUF_SIZE) @@ -632,11 +550,11 @@ def _load_from_pack( break md5.update(data) hash = md5.hexdigest() - if Path(filepath).suffix == ".zip": - with ZipFile(filepath, "r") as zipobj: + if Path(filepath).suffix == '.zip': + with ZipFile(filepath, 'r') as zipobj: zipobj.extractall(str(folder)) - elif Path(filepath).suffix == ".tar": - shutil.unpack_archive(filepath, str(folder), "tar") + elif Path(filepath).suffix == '.tar': + shutil.unpack_archive(filepath, str(folder), 'tar') else: raise ValueError("File is not a zip or tar file") else: @@ -644,7 +562,8 @@ def _load_from_pack( ome = from_xml(folder / "transfer.xml") return hash, ome, folder - def _create_image_map(self, ome: OME) -> Tuple[OME, DefaultDict, List[str]]: + def _create_image_map(self, ome: OME + ) -> Tuple[OME, DefaultDict, List[str]]: if not (type(ome) is OME): raise TypeError("XML is not valid OME format") img_map = DefaultDict(list) @@ -652,15 +571,14 @@ def _create_image_map(self, ome: OME) -> Tuple[OME, DefaultDict, List[str]]: newome = copy.deepcopy(ome) map_ref_ids = [] for ann in ome.structured_annotations: - if ( - int(ann.id.split(":")[-1]) < 0 - and isinstance(ann, CommentAnnotation) - and ann.namespace - ): + if int(ann.id.split(":")[-1]) < 0 \ + and isinstance(ann, CommentAnnotation) \ + and ann.namespace: if ann.namespace.split(":")[0] == "Image": map_ref_ids.append(ann.id) - img_map[ann.value].append(int(ann.namespace.split(":")[-1])) - if ann.value.endswith("mock_folder"): + img_map[ann.value].append(int( + ann.namespace.split(":")[-1])) + if ann.value.endswith('mock_folder'): filelist.append(ann.value.rstrip("mock_folder")) else: filelist.append(ann.value) @@ -670,28 +588,23 @@ def _create_image_map(self, ome: OME) -> Tuple[OME, DefaultDict, List[str]]: if ref.id in map_ref_ids: i.annotation_refs.remove(ref) filelist = list(set(filelist)) - img_map = DefaultDict(list, {x: sorted(img_map[x]) for x in img_map.keys()}) + img_map = DefaultDict(list, {x: sorted(img_map[x]) + for x in img_map.keys()}) return newome, img_map, filelist - def _import_files( - self, - folder: Path, - filelist: List[str], - ln_s: bool, - skip: str, - gateway: BlitzGateway, - ) -> dict: + def _import_files(self, folder: Path, filelist: List[str], ln_s: bool, + skip: str, gateway: BlitzGateway) -> dict: cli = CLI() cli.loadplugins() dest_map = {} - curr_folder = str(Path(".").resolve()) + curr_folder = str(Path('.').resolve()) for filepath in filelist: - dest_path = str(os.path.join(curr_folder, folder, ".", filepath)) - command = ["import", dest_path] + dest_path = str(os.path.join(curr_folder, folder, '.', filepath)) + command = ['import', dest_path] if ln_s: - command.append("--transfer=ln_s") + command.append('--transfer=ln_s') if skip: - command.extend(["--skip", skip]) + command.extend(['--skip', skip]) cli.invoke(command) img_ids = self._get_image_ids(dest_path, gateway) dest_map[dest_path] = img_ids @@ -718,16 +631,16 @@ def _get_image_ids(self, file_path: str, conn: BlitzGateway) -> List[str]: """ q = conn.getQueryService() params = Parameters() - path_query = str(file_path).strip("/") - params.map = {"cpath": rstring("%s%%" % path_query)} + path_query = str(file_path).strip('/') + params.map = {"cpath": rstring('%s%%' % path_query)} results = q.projection( "SELECT i.id FROM Image i" " JOIN i.fileset fs" " JOIN fs.usedFiles u" " WHERE u.clientPath LIKE :cpath", params, - conn.SERVICE_OPTS, - ) + conn.SERVICE_OPTS + ) all_image_ids = list(set(sorted([r[0].val for r in results]))) image_ids = [] for img_id in all_image_ids: @@ -738,15 +651,15 @@ def _get_image_ids(self, file_path: str, conn: BlitzGateway) -> List[str]: is_annotated = False for ann in anns: ann_content = conn.getObject("MapAnnotation", ann) - if ann_content.getNs() == "openmicroscopy.org/cli/transfer": + if ann_content.getNs() == \ + 'openmicroscopy.org/cli/transfer': is_annotated = True if not is_annotated: image_ids.append(img_id) return image_ids - def _make_image_map( - self, source_map: dict, dest_map: dict, conn: Optional[BlitzGateway] = None - ) -> dict: + def _make_image_map(self, source_map: dict, dest_map: dict, + conn: Optional[BlitzGateway] = None) -> dict: # using both source and destination file-to-image-id maps, # map image IDs between source and destination src_dict = DefaultDict(list) @@ -761,10 +674,10 @@ def _make_image_map( for k, v in dest_map.items(): newkey = k.split("/./")[-1] dest_dict[newkey].extend(v) - src_dict = DefaultDict(list, {x: sorted(src_dict[x]) for x in src_dict.keys()}) - dest_dict = DefaultDict( - list, {x: sorted(dest_dict[x]) for x in dest_dict.keys()} - ) + src_dict = DefaultDict(list, {x: sorted(src_dict[x]) + for x in src_dict.keys()}) + dest_dict = DefaultDict(list, {x: sorted(dest_dict[x]) + for x in dest_dict.keys()}) for src_k in src_dict.keys(): src_v = src_dict[src_k] if src_k in dest_dict.keys(): @@ -778,7 +691,8 @@ def _make_image_map( anns = 0 for j in img_obj.listAnnotations(): ns = j.getNs() - if ns.startswith("openmicroscopy.org/cli/transfer"): + if ns.startswith( + "openmicroscopy.org/cli/transfer"): anns += 1 if not anns: clean_dest.append(i) @@ -789,7 +703,8 @@ def _make_image_map( return imgmap def __prepare(self, args): - populate_xml_folder(args.folder, args.filelist, self.gateway, self.session) + populate_xml_folder(args.folder, args.filelist, self.gateway, + self.session) return From e2095bed0d4849f55e7d56529cdbe07d78460da3 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 6 Dec 2023 09:09:09 +0100 Subject: [PATCH 03/18] fix too many blank lines --- test/integration/test_transfer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index 236c194..79cf351 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -188,7 +188,6 @@ def test_pack_special(self, target_name, tmpdir): assert len(f.getmembers()) == 6 self.delete_all() - @pytest.mark.parametrize('target_name', sorted(SUPPORTED)) def test_pack_server(self, target_name, tmpdir): if target_name == "datasetid" or target_name == "projectid" or\ @@ -209,8 +208,6 @@ def test_pack_server(self, target_name, tmpdir): self.cli.invoke(args, strict=True) self.delete_all() - - @pytest.mark.parametrize('folder_name', TEST_FOLDERS) def test_unpack_folder(self, folder_name): self.args += ["unpack", "--folder", folder_name] From e9fb55027eafde275073d0f72322cd3f586dabeb Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 6 Dec 2023 10:00:02 +0100 Subject: [PATCH 04/18] changes cli option to avoid collision --- README.md | 8 ++++++++ src/omero_cli_transfer.py | 16 ++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 72f6536..df129b3 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,13 @@ about the files (name, mimetype). `--metadata` allows you to specify which transfer metadata will be saved in `transfer.xml` as possible MapAnnotation values to the images. Defaults to image ID, timestamp, software version, source hostname, md5, source username, source group. +<<<<<<< HEAD `--plugin` allows you to export omero data to a desired format by using an external plugin. See for example the [arc plugin](https://github.com/cmohl2013/omero-arc), which exports omero projects to ARC repositories. `--server` creates the transfer.xml file but does not copy data +======= +`--metadata_only` creates the transfer.xml file but does not copy data +>>>>>>> ab2fbd3 (changes cli option to avoid collision) or generate an archive. The last cli argument is the path where the `transfer.xml` file will be written @@ -74,8 +78,12 @@ omero transfer pack Image:123 transfer_pack.tar omero transfer pack --zip Image:123 transfer_pack.zip omero transfer pack Dataset:1111 /home/user/new_folder/new_pack.tar omero transfer pack 999 tarfile.tar # equivalent to Project:999 +<<<<<<< HEAD omero transfer pack --plugin arc Project:999 path/to/my/arc/repo omero transfer pack --server Dataset:1111 /home/user/new_folder +======= +omero transfer pack --metadata_only Dataset:1111 /home/user/new_folder +>>>>>>> ab2fbd3 (changes cli option to avoid collision) ``` ## `omero transfer unpack` diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index b978cc1..7a4a2a4 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -85,6 +85,10 @@ orig_group`), other options are `none`, `img_id`, `timestamp`, `software`, `version`, `md5`, `hostname`, `db_id`, `orig_user`, `orig_group`. +--metadata_only creates the transfer.xml file but does not copy data +or generate an archive. The last cli argument is the path where the `transfer.xml` +file will be written + Examples: omero transfer pack Image:123 transfer_pack.tar omero transfer pack --zip Image:123 transfer_pack.zip @@ -225,7 +229,7 @@ def _configure(self, parser): type=str) pack.add_argument("filepath", type=str, help=file_help) pack.add_argument( - "--server", + "--metadata_only", help="Only generate the xml file, don't create the archive" ) @@ -401,8 +405,8 @@ def __pack(self, args): raise ValueError("Single plate or screen cannot be " "packaged in human-readable format") - if args.server and args.simple: - raise ValueError("The --server and --simple options are " + if args.metadata_only and args.simple: + raise ValueError("The --metadata_only and --simple options are " "incompatible") if isinstance(args.object, Image): @@ -431,11 +435,11 @@ def __pack(self, args): " permissions for current user.") print("Populating xml...") tar_path = Path(args.filepath) - if not args.server: + if not args.metadata_only: folder = str(tar_path) + "_folder" else: folder = os.path.splitext(tar_path)[0] - print("Output will be written to {folder}") + print(f"Output will be written to {folder}") os.makedirs(folder, mode=DIR_PERM, exist_ok=True) if args.barchive: @@ -451,7 +455,7 @@ def __pack(self, args): args.figure, self.metadata) - if not args.server: + if not args.metadata_only: print("Starting file copy...") self._copy_files(path_id_dict, folder, self.gateway) From a7653b12a09a5e94a547f02cd5303d85835cf2eb Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 6 Dec 2023 10:04:16 +0100 Subject: [PATCH 05/18] line length --- src/omero_cli_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 7a4a2a4..b6445c0 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -86,8 +86,8 @@ `version`, `md5`, `hostname`, `db_id`, `orig_user`, `orig_group`. --metadata_only creates the transfer.xml file but does not copy data -or generate an archive. The last cli argument is the path where the `transfer.xml` -file will be written +or generate an archive. The last cli argument is the path where +the `transfer.xml` file will be written Examples: omero transfer pack Image:123 transfer_pack.tar From e5e4593c411ab9a2f1e9b29857bdaf313a6ea944 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 6 Dec 2023 10:19:57 +0100 Subject: [PATCH 06/18] adds store_true to add_argument --- src/omero_cli_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index b6445c0..e013da6 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -230,8 +230,8 @@ def _configure(self, parser): pack.add_argument("filepath", type=str, help=file_help) pack.add_argument( "--metadata_only", - help="Only generate the xml file, don't create the archive" - ) + help="Only generate the xml file, don't create the archive", + action="store_true") file_help = ("Path to where the zip file is saved") unpack.add_argument("filepath", type=str, help=file_help) From 4339f66e35ad8d38e7966292d6fbdaa673c57054 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 6 Dec 2023 10:37:03 +0100 Subject: [PATCH 07/18] rename tests --- test/integration/test_transfer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index 79cf351..bb080a6 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -189,20 +189,20 @@ def test_pack_special(self, target_name, tmpdir): self.delete_all() @pytest.mark.parametrize('target_name', sorted(SUPPORTED)) - def test_pack_server(self, target_name, tmpdir): + def test_pack_metadata_only(self, target_name, tmpdir): if target_name == "datasetid" or target_name == "projectid" or\ target_name == "idonly" or target_name == "imageid": self.create_image(target_name=target_name) elif target_name == "plateid" or target_name == "screenid": self.create_plate(target_name=target_name) target = getattr(self, target_name) - args = self.args + ["pack", target, "--server", + args = self.args + ["pack", target, "--metadata_only", str(tmpdir)] self.cli.invoke(args, strict=True) assert os.path.exists(str(tmpdir / 'transfer.xml')) assert os.path.getsize(str(tmpdir / 'transfer.xml')) > 0 - args = self.args + ["pack", target, "--server", "--simple", + args = self.args + ["pack", target, "--metadata_only", "--simple", str(tmpdir)] with pytest.raises(ValueError): self.cli.invoke(args, strict=True) From dd5f53048002dcdf6bf4e137fe2e11319c7c767e Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Mon, 11 Dec 2023 09:56:14 +0100 Subject: [PATCH 08/18] changes cli argument to --- README.md | 8 ++++++++ src/omero_cli_transfer.py | 30 +++++++++++++++++++++--------- test/integration/test_transfer.py | 4 ++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index df129b3..fb123c7 100644 --- a/README.md +++ b/README.md @@ -104,11 +104,19 @@ effectively merging the "new" unpacked entities with existing ones. `--metadata` allows you to specify which transfer metadata will be used from `transfer.xml` as MapAnnotation values to the images. Fields that do not exist on `transfer.xml` will be ignored. Defaults to image ID, timestamp, software version, source hostname, md5, source username, source group. +`--binaries` allows to specify whether to archive binary data +(e.g images, ROIs, FileAnnotations) or only create the transfer.xml. +Default is `all` and will create the archive. With `none`, only the `transfer.xml` +file is created, in which case the last cli argument is the path where +the `transfer.xml` file will be written. + Examples: ``` omero transfer unpack transfer_pack.zip omero transfer unpack --output /home/user/optional_folder --ln_s omero transfer unpack --folder /home/user/unpacked_folder/ +omero transfer pack --binaries none Dataset:1111 /home/user/new_folder/ +omero transfer pack --binaries all Dataset:1111 /home/user/new_folder/new_pack.tar ``` ## `omero transfer prepare` diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index e013da6..bf9feb6 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -85,9 +85,11 @@ orig_group`), other options are `none`, `img_id`, `timestamp`, `software`, `version`, `md5`, `hostname`, `db_id`, `orig_user`, `orig_group`. ---metadata_only creates the transfer.xml file but does not copy data -or generate an archive. The last cli argument is the path where -the `transfer.xml` file will be written +--binaries allows to specify whether to archive binary data +(e.g images, ROIs, FileAnnotations) or only create the transfer.xml. +Default is `all` and will create the archive. WIth `none`, only the `transfer.xml` +file is created, in which case the last cli argument is the path where +the `transfer.xml` file will be written. Examples: omero transfer pack Image:123 transfer_pack.tar @@ -95,6 +97,8 @@ omero transfer pack Dataset:1111 /home/user/new_folder/new_pack.tar omero transfer pack 999 tarfile.tar # equivalent to Project:999 omero transfer pack 1 transfer_pack.tar --metadata img_id version db_id +omero transfer pack --binaries none Dataset:1111 /home/user/new_folder/ +omero transfer pack --binaries all Dataset:1111 /home/user/new_folder/new_pack.tar """) UNPACK_HELP = ("""Unpacks a transfer packet into an OMERO hierarchy. @@ -229,8 +233,12 @@ def _configure(self, parser): type=str) pack.add_argument("filepath", type=str, help=file_help) pack.add_argument( - "--metadata_only", - help="Only generate the xml file, don't create the archive", + "--binaries", + choices=["all", "none"], + default="all", + help="With `--binaries none`, only generate the metadata file " + "(transfer.xml or ro-crate-metadata.json). " + "With `--binaries all` (the default), pixel data and annotation are saved.", action="store_true") file_help = ("Path to where the zip file is saved") @@ -405,8 +413,8 @@ def __pack(self, args): raise ValueError("Single plate or screen cannot be " "packaged in human-readable format") - if args.metadata_only and args.simple: - raise ValueError("The --metadata_only and --simple options are " + if args.binaries == "all" and args.simple: + raise ValueError("The `--binaries all` and `--simple` options are " "incompatible") if isinstance(args.object, Image): @@ -435,7 +443,7 @@ def __pack(self, args): " permissions for current user.") print("Populating xml...") tar_path = Path(args.filepath) - if not args.metadata_only: + if args.binaries == "all": folder = str(tar_path) + "_folder" else: folder = os.path.splitext(tar_path)[0] @@ -455,7 +463,7 @@ def __pack(self, args): args.figure, self.metadata) - if not args.metadata_only: + if args.binaries == "all": print("Starting file copy...") self._copy_files(path_id_dict, folder, self.gateway) @@ -469,6 +477,7 @@ def __pack(self, args): print(f"Creating RO-Crate metadata at {md_fp}.") populate_rocrate(src_datatype, ome, os.path.splitext(tar_path)[0], path_id_dict, folder) +<<<<<<< HEAD if args.plugin: """ Plugins for omero-cli-transfer can be created by providing @@ -499,6 +508,9 @@ def __pack(self, args): image_filenames_mapping=path_id_dict, conn=self.gateway) elif not args.server: +======= + elif args.binaries == "all": +>>>>>>> 2d1e011 (changes cli argument to) self._package_files(os.path.splitext(tar_path)[0], args.zip, folder) print("Cleaning up...") diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index bb080a6..68d969c 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -196,13 +196,13 @@ def test_pack_metadata_only(self, target_name, tmpdir): elif target_name == "plateid" or target_name == "screenid": self.create_plate(target_name=target_name) target = getattr(self, target_name) - args = self.args + ["pack", target, "--metadata_only", + args = self.args + ["pack", target, "--binaries", "none", str(tmpdir)] self.cli.invoke(args, strict=True) assert os.path.exists(str(tmpdir / 'transfer.xml')) assert os.path.getsize(str(tmpdir / 'transfer.xml')) > 0 - args = self.args + ["pack", target, "--metadata_only", "--simple", + args = self.args + ["pack", target, "--binaries", "none", "--simple", str(tmpdir)] with pytest.raises(ValueError): self.cli.invoke(args, strict=True) From 104508266f1923e89825cb9bedeb99cfabdb3613 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Mon, 11 Dec 2023 10:08:06 +0100 Subject: [PATCH 09/18] fix line lengths --- src/omero_cli_transfer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index bf9feb6..4fda006 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +# !/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (C) 2022 The Jackson Laboratory # All rights reserved. @@ -87,9 +87,10 @@ --binaries allows to specify whether to archive binary data (e.g images, ROIs, FileAnnotations) or only create the transfer.xml. -Default is `all` and will create the archive. WIth `none`, only the `transfer.xml` -file is created, in which case the last cli argument is the path where -the `transfer.xml` file will be written. +Default is `all` and will create the archive. +With `none`, only the `transfer.xml` file is created, in which case +the last cli argument is the path where the `transfer.xml` file +will be written. Examples: omero transfer pack Image:123 transfer_pack.tar @@ -98,7 +99,7 @@ omero transfer pack 999 tarfile.tar # equivalent to Project:999 omero transfer pack 1 transfer_pack.tar --metadata img_id version db_id omero transfer pack --binaries none Dataset:1111 /home/user/new_folder/ -omero transfer pack --binaries all Dataset:1111 /home/user/new_folder/new_pack.tar +omero transfer pack --binaries all Dataset:1111 /home/user/new_folder/pack.tar """) UNPACK_HELP = ("""Unpacks a transfer packet into an OMERO hierarchy. @@ -234,11 +235,12 @@ def _configure(self, parser): pack.add_argument("filepath", type=str, help=file_help) pack.add_argument( "--binaries", - choices=["all", "none"], + choices=["all", "none"], default="all", help="With `--binaries none`, only generate the metadata file " "(transfer.xml or ro-crate-metadata.json). " - "With `--binaries all` (the default), pixel data and annotation are saved.", + "With `--binaries all` (the default), both pixel data " + "and annotation are saved.", action="store_true") file_help = ("Path to where the zip file is saved") From fe3ab5711d2cd757c48f89b7b3b9f5ff01ee5b65 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Mon, 11 Dec 2023 10:14:58 +0100 Subject: [PATCH 10/18] removes --- src/omero_cli_transfer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 4fda006..709203a 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -240,8 +240,7 @@ def _configure(self, parser): help="With `--binaries none`, only generate the metadata file " "(transfer.xml or ro-crate-metadata.json). " "With `--binaries all` (the default), both pixel data " - "and annotation are saved.", - action="store_true") + "and annotation are saved.") file_help = ("Path to where the zip file is saved") unpack.add_argument("filepath", type=str, help=file_help) From ccb1cad1f3cb9b8cc8f22299abc8d50d0d174f68 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Mon, 11 Dec 2023 10:36:58 +0100 Subject: [PATCH 11/18] wrong valuerror test --- src/omero_cli_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 709203a..89644ec 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -414,8 +414,8 @@ def __pack(self, args): raise ValueError("Single plate or screen cannot be " "packaged in human-readable format") - if args.binaries == "all" and args.simple: - raise ValueError("The `--binaries all` and `--simple` options are " + if (args.binaries == "none") and args.simple: + raise ValueError("The `--binaries none` and `--simple` options are " "incompatible") if isinstance(args.object, Image): From cf3bf83e32b8d3a9a803cd72839850f3a7427aab Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Mon, 11 Dec 2023 10:41:28 +0100 Subject: [PATCH 12/18] line length (sorry) --- src/omero_cli_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 89644ec..3213e20 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -415,8 +415,8 @@ def __pack(self, args): "packaged in human-readable format") if (args.binaries == "none") and args.simple: - raise ValueError("The `--binaries none` and `--simple` options are " - "incompatible") + raise ValueError("The `--binaries none` and `--simple` options " + "are incompatible") if isinstance(args.object, Image): src_datatype, src_dataid = "Image", args.object.id From d2dff351716a3708896205d69e1a3a34fd7a0dd2 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Tue, 9 Jan 2024 10:49:46 +0100 Subject: [PATCH 13/18] fix readme (doc was on the wrong section) --- README.md | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index fb123c7..f7714dd 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,13 @@ about the files (name, mimetype). `--metadata` allows you to specify which transfer metadata will be saved in `transfer.xml` as possible MapAnnotation values to the images. Defaults to image ID, timestamp, software version, source hostname, md5, source username, source group. -<<<<<<< HEAD `--plugin` allows you to export omero data to a desired format by using an external plugin. See for example the [arc plugin](https://github.com/cmohl2013/omero-arc), which exports omero projects to ARC repositories. -`--server` creates the transfer.xml file but does not copy data -======= -`--metadata_only` creates the transfer.xml file but does not copy data ->>>>>>> ab2fbd3 (changes cli option to avoid collision) -or generate an archive. The last cli argument is the path where the `transfer.xml` -file will be written +`--binaries` allows to specify whether to archive binary data +(e.g images, ROIs, FileAnnotations) or only create the transfer.xml. +Default is `all` and will create the archive. With `none`, only the `transfer.xml` +file is created, in which case the last cli argument is the path where +the `transfer.xml` file will be written. Examples: @@ -78,12 +76,9 @@ omero transfer pack Image:123 transfer_pack.tar omero transfer pack --zip Image:123 transfer_pack.zip omero transfer pack Dataset:1111 /home/user/new_folder/new_pack.tar omero transfer pack 999 tarfile.tar # equivalent to Project:999 -<<<<<<< HEAD omero transfer pack --plugin arc Project:999 path/to/my/arc/repo -omero transfer pack --server Dataset:1111 /home/user/new_folder -======= -omero transfer pack --metadata_only Dataset:1111 /home/user/new_folder ->>>>>>> ab2fbd3 (changes cli option to avoid collision) +omero transfer pack --binaries none Dataset:1111 /home/user/new_folder/ +omero transfer pack --binaries all Dataset:1111 /home/user/new_folder/new_pack.tar ``` ## `omero transfer unpack` @@ -104,19 +99,11 @@ effectively merging the "new" unpacked entities with existing ones. `--metadata` allows you to specify which transfer metadata will be used from `transfer.xml` as MapAnnotation values to the images. Fields that do not exist on `transfer.xml` will be ignored. Defaults to image ID, timestamp, software version, source hostname, md5, source username, source group. -`--binaries` allows to specify whether to archive binary data -(e.g images, ROIs, FileAnnotations) or only create the transfer.xml. -Default is `all` and will create the archive. With `none`, only the `transfer.xml` -file is created, in which case the last cli argument is the path where -the `transfer.xml` file will be written. - Examples: ``` omero transfer unpack transfer_pack.zip omero transfer unpack --output /home/user/optional_folder --ln_s omero transfer unpack --folder /home/user/unpacked_folder/ -omero transfer pack --binaries none Dataset:1111 /home/user/new_folder/ -omero transfer pack --binaries all Dataset:1111 /home/user/new_folder/new_pack.tar ``` ## `omero transfer prepare` From 86c17b190d64e1754cb49c4c29aa4c56a11c3034 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 24 Jan 2024 09:53:34 +0100 Subject: [PATCH 14/18] update .omero, some test failing --- .omero | 1 - .omeroci/cli-build | 10 +--------- .omeroci/py-setup | 6 +++++- LICENSE | 0 README.md | 0 requirements.txt | 0 setup.py | 0 test/integration/test_transfer.py | 2 +- 8 files changed, 7 insertions(+), 12 deletions(-) delete mode 160000 .omero mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 requirements.txt mode change 100644 => 100755 setup.py diff --git a/.omero b/.omero deleted file mode 160000 index 3824e85..0000000 --- a/.omero +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3824e8568d3eb1352eca3a413811b7d665fb3aef diff --git a/.omeroci/cli-build b/.omeroci/cli-build index ed09e48..32bf677 100755 --- a/.omeroci/cli-build +++ b/.omeroci/cli-build @@ -11,15 +11,7 @@ cd $TARGET GUESS=${PWD#*omero-cli-*} PLUGIN=${PLUGIN:-$GUESS} -export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:/bin/java::") -export JAVA_LD_LIBRARY_PATH="$JAVA_HOME/lib/:$JAVA_HOME/lib/server" -export CONDA_BACKUP_JAVA_HOME=$JAVA_HOME -export CONDA_BACKUP_JAVA_LD_LIBRARY_PATH=$JAVA_LD_LIBRARY_PATH - -source /tmp/miniconda/bin/activate -conda init -conda activate omero - export OMERO_DIST=${OMERO_DIST:-/opt/omero/server/OMERO.server} omero $PLUGIN -h + python setup.py test -t test -i ${OMERO_DIST}/etc/ice.config -vs diff --git a/.omeroci/py-setup b/.omeroci/py-setup index cfcd089..a743327 100755 --- a/.omeroci/py-setup +++ b/.omeroci/py-setup @@ -8,6 +8,7 @@ set -x TARGET=${TARGET:-..} PLUGIN=${PLUGIN:-} +<<<<<<< HEAD:.omeroci/py-setup export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:/bin/java::") export JAVA_LD_LIBRARY_PATH="$JAVA_HOME/lib/:$JAVA_HOME/lib/server" export CONDA_BACKUP_JAVA_HOME=$JAVA_HOME @@ -21,8 +22,11 @@ conda activate omero conda install -c conda-forge zeroc-ice=3.6.5 conda install -c bioconda bftools +======= +>>>>>>> be5209a (update .omero, some test failing):.omero/py-setup cd $TARGET cd $(setup_dir) -pip install . +python setup.py sdist +pip install -U dist/*.tar.gz python setup.py clean rm -rf dist *egg-info src/*egg-info diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/test/integration/test_transfer.py b/test/integration/test_transfer.py index 68d969c..e609055 100644 --- a/test/integration/test_transfer.py +++ b/test/integration/test_transfer.py @@ -205,7 +205,7 @@ def test_pack_metadata_only(self, target_name, tmpdir): args = self.args + ["pack", target, "--binaries", "none", "--simple", str(tmpdir)] with pytest.raises(ValueError): - self.cli.invoke(args, strict=True) + self.cli.invoke(args, strict=True) self.delete_all() @pytest.mark.parametrize('folder_name', TEST_FOLDERS) From 335464db2effeb83ec3375d803ea8c1022111bc3 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 24 Jan 2024 10:38:28 +0100 Subject: [PATCH 15/18] remove spurious merge chars, without ruff formating --- src/omero_cli_transfer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/omero_cli_transfer.py b/src/omero_cli_transfer.py index 3213e20..1d9fded 100644 --- a/src/omero_cli_transfer.py +++ b/src/omero_cli_transfer.py @@ -478,7 +478,6 @@ def __pack(self, args): print(f"Creating RO-Crate metadata at {md_fp}.") populate_rocrate(src_datatype, ome, os.path.splitext(tar_path)[0], path_id_dict, folder) -<<<<<<< HEAD if args.plugin: """ Plugins for omero-cli-transfer can be created by providing @@ -508,10 +507,7 @@ def __pack(self, args): tmp_path=Path(folder), image_filenames_mapping=path_id_dict, conn=self.gateway) - elif not args.server: -======= elif args.binaries == "all": ->>>>>>> 2d1e011 (changes cli argument to) self._package_files(os.path.splitext(tar_path)[0], args.zip, folder) print("Cleaning up...") From 23e6427c441072f7d0534ff7a37635af0f27e4b2 Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 24 Jan 2024 10:53:09 +0100 Subject: [PATCH 16/18] merge chores --- .omeroci/py-setup | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.omeroci/py-setup b/.omeroci/py-setup index a743327..3a331cd 100755 --- a/.omeroci/py-setup +++ b/.omeroci/py-setup @@ -8,7 +8,6 @@ set -x TARGET=${TARGET:-..} PLUGIN=${PLUGIN:-} -<<<<<<< HEAD:.omeroci/py-setup export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:/bin/java::") export JAVA_LD_LIBRARY_PATH="$JAVA_HOME/lib/:$JAVA_HOME/lib/server" export CONDA_BACKUP_JAVA_HOME=$JAVA_HOME @@ -21,12 +20,3 @@ conda create -n omero python=3.9 conda activate omero conda install -c conda-forge zeroc-ice=3.6.5 conda install -c bioconda bftools - -======= ->>>>>>> be5209a (update .omero, some test failing):.omero/py-setup -cd $TARGET -cd $(setup_dir) -python setup.py sdist -pip install -U dist/*.tar.gz -python setup.py clean -rm -rf dist *egg-info src/*egg-info From 96f2f9b64a500ef6332cc45389cda46ac8a6c93a Mon Sep 17 00:00:00 2001 From: Guillaume Gay Date: Wed, 24 Jan 2024 11:49:18 +0100 Subject: [PATCH 17/18] restore wrongly deleted lines --- .omeroci/cli-build | 9 +++++++++ .omeroci/py-setup | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/.omeroci/cli-build b/.omeroci/cli-build index 32bf677..8ef8e37 100755 --- a/.omeroci/cli-build +++ b/.omeroci/cli-build @@ -11,6 +11,15 @@ cd $TARGET GUESS=${PWD#*omero-cli-*} PLUGIN=${PLUGIN:-$GUESS} +export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:/bin/java::") +export JAVA_LD_LIBRARY_PATH="$JAVA_HOME/lib/:$JAVA_HOME/lib/server" +export CONDA_BACKUP_JAVA_HOME=$JAVA_HOME +export CONDA_BACKUP_JAVA_LD_LIBRARY_PATH=$JAVA_LD_LIBRARY_PATH + +source /tmp/miniconda/bin/activate +conda init +conda activate omero + export OMERO_DIST=${OMERO_DIST:-/opt/omero/server/OMERO.server} omero $PLUGIN -h diff --git a/.omeroci/py-setup b/.omeroci/py-setup index 3a331cd..cfcd089 100755 --- a/.omeroci/py-setup +++ b/.omeroci/py-setup @@ -20,3 +20,9 @@ conda create -n omero python=3.9 conda activate omero conda install -c conda-forge zeroc-ice=3.6.5 conda install -c bioconda bftools + +cd $TARGET +cd $(setup_dir) +pip install . +python setup.py clean +rm -rf dist *egg-info src/*egg-info From f0736954cedf34a560fbc02c7c53f3a61e906698 Mon Sep 17 00:00:00 2001 From: Erick Martins Ratamero Date: Wed, 24 Jan 2024 09:37:49 -0500 Subject: [PATCH 18/18] making sure submodule is still there --- .omero | 1 + 1 file changed, 1 insertion(+) create mode 160000 .omero diff --git a/.omero b/.omero new file mode 160000 index 0000000..3824e85 --- /dev/null +++ b/.omero @@ -0,0 +1 @@ +Subproject commit 3824e8568d3eb1352eca3a413811b7d665fb3aef