diff --git a/api/src/main/java/org/fairdatapipeline/api/CodeRepo.java b/api/src/main/java/org/fairdatapipeline/api/CodeRepo.java index fea5e0a..d774a70 100644 --- a/api/src/main/java/org/fairdatapipeline/api/CodeRepo.java +++ b/api/src/main/java/org/fairdatapipeline/api/CodeRepo.java @@ -1,6 +1,5 @@ package org.fairdatapipeline.api; -import java.net.URL; import java.util.List; import org.fairdatapipeline.dataregistry.restclient.APIURL; @@ -15,7 +14,7 @@ class CodeRepo { CodeRepo( String latest_commit, - URL repo_url, + String repo_url, String description, List authors, Coderun coderun) { diff --git a/api/src/main/java/org/fairdatapipeline/api/Coderun.java b/api/src/main/java/org/fairdatapipeline/api/Coderun.java index 0128de3..419ef52 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Coderun.java +++ b/api/src/main/java/org/fairdatapipeline/api/Coderun.java @@ -3,9 +3,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URI; -import java.net.URL; import java.nio.file.*; import java.time.LocalDateTime; import java.util.*; @@ -237,18 +235,11 @@ private void prepare_code_run() { this.registryCode_run.setSubmission_script(this.script_object.getUrl()); String latest_commit = this.config.run_metadata().latest_commit().orElse(""); String remote_repo = this.config.run_metadata().remote_repo().orElse(""); - URL remote_repo_url; - try { - remote_repo_url = new URL(remote_repo); - } catch (MalformedURLException e) { - throw (new ConfigException( - "Remote repo must be a valid URL; (" + remote_repo + " isn't)", e)); - } this.codeRepo = new CodeRepo( latest_commit, - remote_repo_url, + remote_repo, "Analysis / processing script location", this.authors, this); diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java index 9df0b69..9230ce6 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_read.java @@ -4,8 +4,14 @@ import java.nio.file.Path; import java.util.List; import org.fairdatapipeline.distribution.Distribution; +import org.fairdatapipeline.distribution.ImmutableDistribution; +import org.fairdatapipeline.estimate.ImmutableEstimate; import org.fairdatapipeline.file.CleanableFileChannel; +import org.fairdatapipeline.parameters.ImmutableBoolList; +import org.fairdatapipeline.parameters.ImmutableNumberList; +import org.fairdatapipeline.parameters.ImmutableStringList; import org.fairdatapipeline.parameters.ReadComponent; +import org.fairdatapipeline.samples.ImmutableSamples; /** * This represents an object_component to read from (or raise issues with) An object_component @@ -77,7 +83,11 @@ public Number readEstimate() { } catch (IOException e) { throw (new RuntimeException("readEstimate() -- IOException trying to read from file", e)); } - return data.getEstimate(); + if (!(data instanceof ImmutableEstimate)) { + throw (new RuntimeException( + "readEstimate() -- this objComponent (" + this.component_name + ") is not an estimate")); + } + return ((ImmutableEstimate) data).getEstimate(); } /** @@ -93,7 +103,70 @@ public Distribution readDistribution() { throw (new RuntimeException( "readDistribution() -- IOException trying to read from file.", e)); } - return data.getDistribution(); + if (!(data instanceof ImmutableDistribution)) { + throw (new RuntimeException( + "readDistribution() -- this objComponent (" + + this.component_name + + ") is not a distribution")); + } + return ((ImmutableDistribution) data).getDistribution(); + } + + /** + * read the Bools that were stored as this component in a TOML file. + * + * @return the Bools object + */ + public List readBools() { + ReadComponent data; + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); + } catch (IOException e) { + throw (new RuntimeException("readBools() -- IOException trying to read from file.", e)); + } + if (!(data instanceof ImmutableBoolList)) { + throw (new RuntimeException( + "readBools() -- this objComponent (" + this.component_name + ") is not a BoolList")); + } + return ((ImmutableBoolList) data).getBools(); + } + + /** + * read the Strings that were stored as this component in a TOML file. + * + * @return the Strings object + */ + public List readStrings() { + ReadComponent data; + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); + } catch (IOException e) { + throw (new RuntimeException("readStrings() -- IOException trying to read from file.", e)); + } + if (!(data instanceof ImmutableStringList)) { + throw (new RuntimeException( + "readStrings() -- this objComponent (" + this.component_name + ") is not a StringList")); + } + return ((ImmutableStringList) data).getStrings(); + } + + /** + * read the Numbers that were stored as this component in a TOML file. + * + * @return the Numbers object + */ + public List readNumbers() { + ReadComponent data; + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + data = this.dp.coderun.parameterDataReader.read(fileChannel, this.component_name); + } catch (IOException e) { + throw (new RuntimeException("readStrings() -- IOException trying to read from file.", e)); + } + if (!(data instanceof ImmutableNumberList)) { + throw (new RuntimeException( + "readNumbers() -- this objComponent (" + this.component_name + ") is not a NumberList")); + } + return ((ImmutableNumberList) data).getNumbers(); } /** @@ -108,7 +181,11 @@ public List readSamples() { } catch (IOException e) { throw (new RuntimeException("readSamples() -- IOException trying to read from file.", e)); } - return data.getSamples(); + if (!(data instanceof ImmutableSamples)) { + throw (new RuntimeException( + "readSamples() -- this objComponent (" + this.component_name + ") is not a samples")); + } + return ((ImmutableSamples) data).getSamples(); } void register_me_in_registry() { diff --git a/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java b/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java index 5204143..109b76c 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java +++ b/api/src/main/java/org/fairdatapipeline/api/Object_component_write.java @@ -7,6 +7,9 @@ import org.fairdatapipeline.distribution.Distribution; import org.fairdatapipeline.estimate.ImmutableEstimate; import org.fairdatapipeline.file.CleanableFileChannel; +import org.fairdatapipeline.parameters.BoolList; +import org.fairdatapipeline.parameters.NumberList; +import org.fairdatapipeline.parameters.StringList; import org.fairdatapipeline.samples.Samples; /** @@ -64,6 +67,9 @@ public CleanableFileChannel writeFileChannel() throws IOException { * @param estimateNumber the number to write. */ public void writeEstimate(Number estimateNumber) { + if (this.been_used) { + throw (new RuntimeException("obj component already written")); + } var estimate = ImmutableEstimate.builder().internalValue(estimateNumber).rng(this.dp.coderun.rng).build(); @@ -72,6 +78,7 @@ public void writeEstimate(Number estimateNumber) { } catch (IOException e) { throw (new RuntimeException("writeEstimate() -- IOException trying to write to file.", e)); } + this.been_used = true; } /** @@ -80,12 +87,67 @@ public void writeEstimate(Number estimateNumber) { * @param distribution the Distribution to write */ public void writeDistribution(Distribution distribution) { + if (this.been_used) { + throw (new RuntimeException("obj component already written")); + } try (CleanableFileChannel fileChannel = this.getFileChannel()) { this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, distribution); } catch (IOException e) { throw (new RuntimeException( "writeDistribution() -- IOException trying to write to file.", e)); } + this.been_used = true; + } + + /** + * write a BoolList, as this named component in the data product. + * + * @param bools the Booleans to write + */ + public void writeBools(BoolList bools) { + if (this.been_used) { + throw (new RuntimeException("obj component already written")); + } + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, bools); + } catch (IOException e) { + throw (new RuntimeException("writeBools() -- IOException trying to write to file.", e)); + } + this.been_used = true; + } + + /** + * write a StringList, as this named component in the data product. + * + * @param strings the Strings to write + */ + public void writeStrings(StringList strings) { + if (this.been_used) { + throw (new RuntimeException("obj component already written")); + } + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, strings); + } catch (IOException e) { + throw (new RuntimeException("writeStrings() -- IOException trying to write to file.", e)); + } + this.been_used = true; + } + + /** + * write NumberList, as this named component in the data product. + * + * @param numbers the Numbers to write + */ + public void writeNumbers(NumberList numbers) { + if (this.been_used) { + throw (new RuntimeException("obj component already written")); + } + try (CleanableFileChannel fileChannel = this.getFileChannel()) { + this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, numbers); + } catch (IOException e) { + throw (new RuntimeException("writeStrings() -- IOException trying to write to file.", e)); + } + this.been_used = true; } /** @@ -94,11 +156,15 @@ public void writeDistribution(Distribution distribution) { * @param samples a Samples object containing the samples */ public void writeSamples(Samples samples) { + if (this.been_used) { + throw (new RuntimeException("obj component already written")); + } try (CleanableFileChannel fileChannel = this.getFileChannel()) { this.dp.coderun.parameterDataWriter.write(fileChannel, this.component_name, samples); } catch (IOException e) { throw (new RuntimeException("writeSamples() -- IOException trying to write to file.", e)); } + this.been_used = true; } void register_me_in_code_run() { diff --git a/api/src/main/java/org/fairdatapipeline/api/Storage_location.java b/api/src/main/java/org/fairdatapipeline/api/Storage_location.java index cf6022c..5220d9a 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Storage_location.java +++ b/api/src/main/java/org/fairdatapipeline/api/Storage_location.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.net.URI; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -14,6 +13,7 @@ /** This is used to store a file or a remote repo to the registry as a RegistryStorage_location. */ class Storage_location { private static final Logger logger = LoggerFactory.getLogger(Storage_location.class); + RegistryStorage_location registryStorage_location; /** @@ -25,8 +25,8 @@ class Storage_location { * we will use the existing one instead of storing this one. * @param coderun link back to the Coderun that created us. */ - Storage_location(URL remote_repo, String latest_commit, Coderun coderun) { - String[] split_repo = Storage_root.url_to_root(remote_repo); + Storage_location(String remote_repo, String latest_commit, Coderun coderun) { + String[] split_repo = Storage_root.gitrepo_to_root(remote_repo); Storage_root storage_root = new Storage_root(URI.create(split_repo[0]), coderun.restClient); create_storagelocation(latest_commit, storage_root, coderun, split_repo[1], null); } diff --git a/api/src/main/java/org/fairdatapipeline/api/Storage_root.java b/api/src/main/java/org/fairdatapipeline/api/Storage_root.java index 76229eb..647d33d 100644 --- a/api/src/main/java/org/fairdatapipeline/api/Storage_root.java +++ b/api/src/main/java/org/fairdatapipeline/api/Storage_root.java @@ -1,15 +1,21 @@ package org.fairdatapipeline.api; import java.net.URI; -import java.net.URL; import java.nio.file.Path; import java.util.Collections; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.fairdatapipeline.dataregistry.content.RegistryStorage_root; import org.fairdatapipeline.dataregistry.restclient.APIURL; import org.fairdatapipeline.dataregistry.restclient.RestClient; /** Retrieve or create the RegistryStorage_root with a given 'root'. */ class Storage_root { + private static Pattern git_repo_url = + Pattern.compile("(\\w+://)(.+@)*([\\w\\d\\.]+)(:[\\d]+){0,1}/*(.*)"); + private static Pattern git_repo_file = Pattern.compile("file://(.*)"); + private static Pattern git_repo_ssh = Pattern.compile("(.+@)([\\w\\d\\.]+):(.*)"); RegistryStorage_root registryStorage_root; /** @@ -47,15 +53,27 @@ Path getPath() { } /** - * split the repository URL into a storage root (proto://authority/ part) and path (/xxx/xxx) part + * split the repository location into a storage root HTTPS: proto://authority/path/to/stuff + * becomes proto://authority AND /path/to/stuff SSH: git@epic.sruc.ac.uk:bboskamp/BTv.git becomes + * git@epic.sruc.ac.uk AND /bboskamp/BTv.git * - * @param url the URL to split up into scheme/authority and path. + * @param repo_location the repository location string to split up into scheme/authority and path. * @return string array of length 2. */ - static String[] url_to_root(URL url) { - String path = url.getPath().substring(1); - String scheme_and_authority_part = - url.toString().substring(0, url.toString().length() - path.length()); - return new String[] {scheme_and_authority_part, path}; + static String[] gitrepo_to_root(String repo_location) { + Matcher m1 = git_repo_url.matcher(repo_location); + if (m1.find()) + return new String[] { + Objects.toString(m1.group(1), "") + + Objects.toString(m1.group(2), "") + + Objects.toString(m1.group(3), "") + + Objects.toString(m1.group(4), ""), + m1.group(5) + }; + m1 = git_repo_file.matcher(repo_location); + if (m1.find()) return new String[] {"file://", m1.group(1)}; + m1 = git_repo_ssh.matcher(repo_location); + if (m1.find()) return new String[] {m1.group(1) + m1.group(2), m1.group(3)}; + return new String[] {}; } } diff --git a/api/src/main/java/org/fairdatapipeline/distribution/Distribution.java b/api/src/main/java/org/fairdatapipeline/distribution/Distribution.java index 73661b8..6bcdb02 100644 --- a/api/src/main/java/org/fairdatapipeline/distribution/Distribution.java +++ b/api/src/main/java/org/fairdatapipeline/distribution/Distribution.java @@ -18,7 +18,7 @@ import org.apache.commons.math3.distribution.RealDistribution; import org.apache.commons.math3.distribution.UniformRealDistribution; import org.apache.commons.math3.random.EmpiricalDistribution; -import org.fairdatapipeline.parameters.Component; +import org.fairdatapipeline.parameters.RngComponent; import org.immutables.value.Value.Auxiliary; import org.immutables.value.Value.Check; import org.immutables.value.Value.Immutable; @@ -33,7 +33,7 @@ TODO support other distributions: https://github.com/ScottishCovidResponse/SCRCIssueTracking/issues/671 */ -public interface Distribution extends Component { +public interface Distribution extends RngComponent { enum DistributionType { gamma(), exponential(), diff --git a/api/src/main/java/org/fairdatapipeline/estimate/Estimate.java b/api/src/main/java/org/fairdatapipeline/estimate/Estimate.java index 41c4f90..59b8c06 100644 --- a/api/src/main/java/org/fairdatapipeline/estimate/Estimate.java +++ b/api/src/main/java/org/fairdatapipeline/estimate/Estimate.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; import org.fairdatapipeline.distribution.Distribution; -import org.fairdatapipeline.parameters.Component; +import org.fairdatapipeline.parameters.RngComponent; import org.immutables.value.Value.Immutable; /** a component to store a plain simple single Number */ @Immutable @JsonDeserialize @JsonSerialize -public interface Estimate extends Component { +public interface Estimate extends RngComponent { /** @return Number - the value that is stored in this component */ @JsonProperty("value") Number internalValue(); diff --git a/api/src/main/java/org/fairdatapipeline/parameters/BoolList.java b/api/src/main/java/org/fairdatapipeline/parameters/BoolList.java new file mode 100644 index 0000000..5a31040 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/parameters/BoolList.java @@ -0,0 +1,19 @@ +package org.fairdatapipeline.parameters; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import org.immutables.value.Value.Immutable; + +@Immutable +@JsonSerialize +@JsonDeserialize +public interface BoolList extends Component { + List bools(); + + @JsonIgnore + default List getBools() { + return bools(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/parameters/Component.java b/api/src/main/java/org/fairdatapipeline/parameters/Component.java index ed14590..6b8ff58 100644 --- a/api/src/main/java/org/fairdatapipeline/parameters/Component.java +++ b/api/src/main/java/org/fairdatapipeline/parameters/Component.java @@ -1,9 +1,3 @@ package org.fairdatapipeline.parameters; -import org.apache.commons.math3.random.RandomGenerator; -import org.immutables.value.Value.Auxiliary; - -public interface Component extends ReadComponent, WriteComponent { - @Auxiliary - RandomGenerator rng(); -} +public interface Component extends ReadComponent, WriteComponent {} diff --git a/api/src/main/java/org/fairdatapipeline/parameters/ComponentsDeserializer.java b/api/src/main/java/org/fairdatapipeline/parameters/ComponentsDeserializer.java index 18c6630..65da584 100644 --- a/api/src/main/java/org/fairdatapipeline/parameters/ComponentsDeserializer.java +++ b/api/src/main/java/org/fairdatapipeline/parameters/ComponentsDeserializer.java @@ -29,9 +29,18 @@ public ComponentsDeserializer(RandomGenerator rng) { private static final Map> typeMapping = Map.of( - "point-estimate", ImmutableEstimate.class, - "distribution", ImmutableDistribution.class, - "samples", ImmutableSamples.class); + "point-estimate", + ImmutableEstimate.class, + "distribution", + ImmutableDistribution.class, + "samples", + ImmutableSamples.class, + "bools", + ImmutableBoolList.class, + "strings", + ImmutableStringList.class, + "numbers", + ImmutableNumberList.class); @Override public Components deserialize(JsonParser jsonParser, DeserializationContext ctxt) @@ -42,15 +51,12 @@ public Components deserialize(JsonParser jsonParser, DeserializationContext ctxt Streams.stream(rootNode.fields()) .map(this::deserializeSingleComponent) .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); - return ImmutableComponents.builder().components(components).build(); } private Entry deserializeSingleComponent(Entry entry) { String key = entry.getKey(); ObjectNode componentNode = entry.getValue().deepCopy(); - componentNode.putPOJO("rng", new Object()); // this is a hack to force rng to populate via - // RandomGeneratorDeserializer.class String type = componentNode.get("type").asText(); componentNode.remove("type"); Class deserializeClass = typeMapping.get(type); @@ -58,7 +64,11 @@ private Entry deserializeSingleComponent(Entry, String> typeMapping = Map.of( - ImmutableEstimate.class, "point-estimate", - ImmutableDistribution.class, "distribution", - ImmutableSamples.class, "samples"); + ImmutableEstimate.class, + "point-estimate", + ImmutableDistribution.class, + "distribution", + ImmutableSamples.class, + "samples", + ImmutableStringList.class, + "strings", + ImmutableBoolList.class, + "bools", + ImmutableNumberList.class, + "numbers"); @Override public void serialize(Components components, JsonGenerator gen, SerializerProvider serializers) diff --git a/api/src/main/java/org/fairdatapipeline/parameters/NumberList.java b/api/src/main/java/org/fairdatapipeline/parameters/NumberList.java new file mode 100644 index 0000000..cfd3312 --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/parameters/NumberList.java @@ -0,0 +1,33 @@ +package org.fairdatapipeline.parameters; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import java.util.stream.Collectors; +import org.immutables.value.Value; +import org.immutables.value.Value.Immutable; + +@Immutable +@JsonSerialize +@JsonDeserialize +public interface NumberList extends Component { + List numbers(); + + @JsonIgnore + default List getNumbers() { + return numbers(); + } + + @Value.Check + default NumberList avoidHeterogeneous() { + // count the number of integers in this list: + int i = (int) numbers().stream().filter((x) -> ((Number) x.intValue()) == x).count(); + if (i != 0 && i < numbers().size()) { + return ImmutableNumberList.builder() + .numbers(numbers().stream().map(Number::doubleValue).collect(Collectors.toList())) + .build(); + } + return this; + } +} diff --git a/api/src/main/java/org/fairdatapipeline/parameters/ReadComponent.java b/api/src/main/java/org/fairdatapipeline/parameters/ReadComponent.java index 88feb3c..400b6e8 100644 --- a/api/src/main/java/org/fairdatapipeline/parameters/ReadComponent.java +++ b/api/src/main/java/org/fairdatapipeline/parameters/ReadComponent.java @@ -1,12 +1,3 @@ package org.fairdatapipeline.parameters; -import java.util.List; -import org.fairdatapipeline.distribution.Distribution; - -public interface ReadComponent { - Number getEstimate(); - - List getSamples(); - - Distribution getDistribution(); -} +public interface ReadComponent {} diff --git a/api/src/main/java/org/fairdatapipeline/parameters/RngComponent.java b/api/src/main/java/org/fairdatapipeline/parameters/RngComponent.java new file mode 100644 index 0000000..102adbe --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/parameters/RngComponent.java @@ -0,0 +1,17 @@ +package org.fairdatapipeline.parameters; + +import java.util.List; +import org.apache.commons.math3.random.RandomGenerator; +import org.fairdatapipeline.distribution.Distribution; +import org.immutables.value.Value; + +public interface RngComponent extends Component { + @Value.Auxiliary + RandomGenerator rng(); + + Number getEstimate(); + + List getSamples(); + + Distribution getDistribution(); +} diff --git a/api/src/main/java/org/fairdatapipeline/parameters/StringList.java b/api/src/main/java/org/fairdatapipeline/parameters/StringList.java new file mode 100644 index 0000000..b52159e --- /dev/null +++ b/api/src/main/java/org/fairdatapipeline/parameters/StringList.java @@ -0,0 +1,19 @@ +package org.fairdatapipeline.parameters; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import org.immutables.value.Value.Immutable; + +@Immutable +@JsonSerialize +@JsonDeserialize +public interface StringList extends Component { + List strings(); + + @JsonIgnore + default List getStrings() { + return strings(); + } +} diff --git a/api/src/main/java/org/fairdatapipeline/samples/Samples.java b/api/src/main/java/org/fairdatapipeline/samples/Samples.java index cf684ac..6835030 100644 --- a/api/src/main/java/org/fairdatapipeline/samples/Samples.java +++ b/api/src/main/java/org/fairdatapipeline/samples/Samples.java @@ -4,15 +4,17 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; +import java.util.stream.Collectors; import org.fairdatapipeline.distribution.Distribution; import org.fairdatapipeline.distribution.ImmutableDistribution; -import org.fairdatapipeline.parameters.Component; +import org.fairdatapipeline.parameters.RngComponent; +import org.immutables.value.Value; import org.immutables.value.Value.Immutable; @Immutable @JsonSerialize @JsonDeserialize -public interface Samples extends Component { +public interface Samples extends RngComponent { List samples(); @JsonIgnore @@ -41,4 +43,17 @@ default Distribution getDistribution() { .rng(rng()) .build(); } + + @Value.Check + default Samples avoidHeterogeneous() { + // count the number of integers in this list: + int i = (int) samples().stream().filter((x) -> ((Number) x.intValue()) == x).count(); + if (i != 0 && i < samples().size()) { + return ImmutableSamples.builder() + .samples(samples().stream().map(Number::doubleValue).collect(Collectors.toList())) + .rng(rng()) + .build(); + } + return this; + } } diff --git a/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java b/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java index 6c9bc30..d648981 100644 --- a/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java +++ b/api/src/test/java/org/fairdatapipeline/api/CoderunIntegrationTest.java @@ -2,6 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -26,6 +27,7 @@ import org.fairdatapipeline.distribution.ImmutableMinMax; import org.fairdatapipeline.distribution.MinMax; import org.fairdatapipeline.file.CleanableFileChannel; +import org.fairdatapipeline.parameters.*; import org.fairdatapipeline.samples.ImmutableSamples; import org.fairdatapipeline.samples.Samples; import org.javatuples.Triplet; @@ -50,6 +52,9 @@ class CoderunIntegrationTest { private final String chickenTestText = "This is a text file about chickens.\nPlease forgive the lack of interesting content in this file."; private Samples samples, samples2, samples3, samples4; + private StringList stringlist1, stringlist2; + private BoolList boollist; + private NumberList numberlist; private Distribution distribution; private Distribution categoricalDistribution; private final Number estimate = 1.0; @@ -57,16 +62,18 @@ class CoderunIntegrationTest { private RestClient restClient; private Path ori_configPath; + private Path ori_configPath2; private Path ori_scriptPath; private Path datastorePath; private String coderun_ts; private Path coderunPath; private Path coderunTSPath; - private String ns; + private String ns, ns2; private String altNamespace; - private Path nsPath; + private Path nsPath, nsPath2; + private Path altNamespacePath; - private Path configPath; + private Path configPath, configPath2; private Path scriptPath; private Path coderuns_txt; private String token; @@ -78,6 +85,9 @@ class CoderunIntegrationTest { void setup_paths() throws URISyntaxException { ori_configPath = Paths.get(Objects.requireNonNull(getClass().getResource("/config-stdapi.yaml")).toURI()); + ori_configPath2 = + Paths.get(Objects.requireNonNull(getClass().getResource("/config_remrepo1.yaml")).toURI()); + ori_scriptPath = Paths.get(Objects.requireNonNull(getClass().getResource("/script.sh")).toURI()); datastorePath = ori_configPath.getParent().resolve("datastore"); @@ -85,10 +95,13 @@ void setup_paths() throws URISyntaxException { coderunPath = datastorePath.resolve("coderun"); coderunTSPath = coderunPath.resolve(coderun_ts); ns = "CoderunTest"; + ns2 = "CoderunTest2"; altNamespace = "alternativeNS"; nsPath = datastorePath.resolve(ns); + nsPath2 = datastorePath.resolve(ns2); altNamespacePath = datastorePath.resolve(altNamespace); configPath = coderunTSPath.resolve("config.yaml"); + configPath2 = coderunTSPath.resolve("config2.yaml"); scriptPath = coderunTSPath.resolve("script.sh"); coderuns_txt = coderunTSPath.resolve("coderuns.txt"); } @@ -98,6 +111,10 @@ void setup_data() { samples2 = ImmutableSamples.builder().addSamples(4, 5, 6).rng(rng).build(); samples3 = ImmutableSamples.builder().addSamples(7, 8, 9).rng(rng).build(); samples4 = ImmutableSamples.builder().addSamples(10, 11, 12).rng(rng).build(); + stringlist1 = ImmutableStringList.builder().addStrings("do", "re", "mi").build(); + stringlist2 = ImmutableStringList.builder().addStrings("just the one").build(); + boollist = ImmutableBoolList.builder().addBools(true).build(); + numberlist = ImmutableNumberList.builder().addNumbers(1, 1.5, 12345.67).build(); csv_data = new ArrayList<>(); csv_data.add(new String[] {"apple", "12", "green"}); @@ -197,6 +214,7 @@ public void cleanup_datastore() throws IOException { delete_directories(); Files.createDirectories(coderunTSPath); Files.copy(ori_configPath, configPath); + Files.copy(ori_configPath2, configPath2); Files.copy(ori_scriptPath, scriptPath); } @@ -204,6 +222,9 @@ void delete_directories() throws IOException { FileUtils.deleteDirectory(coderunTSPath.toFile()); FileUtils.deleteDirectory( nsPath.toFile()); // remove the whole namespace in the local datastore. + FileUtils.deleteDirectory( + nsPath2.toFile()); // remove the whole namespace in the local datastore. + FileUtils.deleteDirectory( altNamespacePath .toFile()); // remove the whole alternative namespace in the local datastore. @@ -325,6 +346,19 @@ void check_last_coderun( @Test @Order(1) + void testWriteEstimateFail() { + String dataProduct = "human/population2"; + String component = "estimate-component"; + try (Coderun coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + Object_component_write oc = dp.getComponent(component); + oc.writeEstimate(estimate); + assertThrows(RuntimeException.class, () -> oc.writeEstimate(estimate)); + } + } + + @Test + @Order(2) void testWriteEstimate() { String dataProduct = "human/population"; String component = "estimate-component"; @@ -338,7 +372,7 @@ void testWriteEstimate() { } @Test - @Order(2) + @Order(3) void testReadEstimate() { String dataProduct = "human/population"; String component = "estimate-component"; @@ -352,7 +386,35 @@ void testReadEstimate() { } @Test - @Order(3) + @Order(4) + void testWriteEstimate_remrep2() { + String dataProduct = "human/population"; + String component = "estimate-component"; + try (Coderun coderun = new Coderun(configPath2, scriptPath, token)) { + Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + Object_component_write oc = dp.getComponent(component); + oc.writeEstimate(estimate); + } + String hash = "a4f9d47dac45639e69a758a8b2d49bf11bbeb262"; + check_last_coderun(null, List.of(new Triplet<>(dataProduct, component, hash))); + } + + @Test + @Order(5) + void testReadEstimate_remrep2() { + String dataProduct = "human/population"; + String component = "estimate-component"; + try (var coderun = new Coderun(configPath2, scriptPath, token)) { + Data_product_read dp = coderun.get_dp_for_read(dataProduct); + Object_component_read oc = dp.getComponent(component); + assertThat(oc.readEstimate()).isEqualTo(estimate); + } + String hash = "a4f9d47dac45639e69a758a8b2d49bf11bbeb262"; + check_last_coderun(List.of(new Triplet<>(dataProduct, component, hash)), null); + } + + @Test + @Order(6) void testWriteDistribution() { String dataProduct = "human/distribution"; String component = "distribution-component"; @@ -366,7 +428,7 @@ void testWriteDistribution() { } @Test - @Order(4) + @Order(7) void testReadDistribution() { String dataProduct = "human/distribution"; String component = "distribution-component"; @@ -380,7 +442,20 @@ void testReadDistribution() { } @Test - @Order(5) + @Order(8) + void testWriteCategoricalDistributionFail() { + String dataProduct = "human/cdistribution2"; + String component = "cdistribution-component"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + Object_component_write oc = dp.getComponent(component); + oc.writeDistribution(categoricalDistribution); + assertThrows(RuntimeException.class, () -> oc.writeDistribution(distribution)); + } + } + + @Test + @Order(9) void testWriteCategoricalDistribution() { String dataProduct = "human/cdistribution"; String component = "cdistribution-component"; @@ -394,7 +469,7 @@ void testWriteCategoricalDistribution() { } @Test - @Order(6) + @Order(10) void testReadCategoricalDistribution() { String dataProduct = "human/cdistribution"; String component = "cdistribution-component"; @@ -408,7 +483,20 @@ void testReadCategoricalDistribution() { } @Test - @Order(7) + @Order(11) + void testWriteSamplesFail() throws RuntimeException { + String dataProduct = "human/samples2"; + String component = "example-samples-w"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + Object_component_write oc = dp.getComponent(component); + oc.writeSamples(samples2); + assertThrows(RuntimeException.class, () -> oc.writeSamples(samples3)); + } + } + + @Test + @Order(12) void testWriteSamples() { String dataProduct = "human/samples"; String component = "example-samples-w"; @@ -422,7 +510,7 @@ void testWriteSamples() { } @Test - @Order(8) + @Order(13) void testReadSamples() { String dataProduct = "human/samples"; String component = "example-samples-w"; @@ -436,7 +524,7 @@ void testReadSamples() { } @Test - @Order(9) + @Order(14) void testWriteSamplesMultipleComponents() { String dataProduct = "human/multicomp"; String component1 = "example-samples-w1"; @@ -459,7 +547,7 @@ void testWriteSamplesMultipleComponents() { } @Test - @Order(10) + @Order(15) void testReadSamplesMultipleComponents() { String dataProduct = "human/multicomp"; String component1 = "example-samples-w1"; @@ -480,7 +568,67 @@ void testReadSamplesMultipleComponents() { } @Test - @Order(11) + @Order(16) + void testWriteAllSortsComponents() { + String dataProduct = "human/allsortscomp"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_write dp = coderun.get_dp_for_write(dataProduct, "toml"); + + dp.getComponent("a").writeSamples(samples); + dp.getComponent("b").writeSamples(samples2); + dp.getComponent("c").writeDistribution(categoricalDistribution); + dp.getComponent("d").writeEstimate(estimate); + dp.getComponent("e").writeStrings(stringlist1); + dp.getComponent("f").writeStrings(stringlist2); + dp.getComponent("g").writeBools(boollist); + dp.getComponent("h").writeNumbers(numberlist); + } + String hash = "68608005664459de456d75941430e7290031ddb5"; + check_last_coderun( + null, + Arrays.asList( + new Triplet<>(dataProduct, "a", hash), + new Triplet<>(dataProduct, "b", hash), + new Triplet<>(dataProduct, "c", hash), + new Triplet<>(dataProduct, "d", hash), + new Triplet<>(dataProduct, "e", hash), + new Triplet<>(dataProduct, "f", hash), + new Triplet<>(dataProduct, "g", hash), + new Triplet<>(dataProduct, "h", hash))); + } + + @Test + @Order(17) + void testReadAllSortsComponents() { + String dataProduct = "human/allsortscomp"; + try (var coderun = new Coderun(configPath, scriptPath, token)) { + Data_product_read dc = coderun.get_dp_for_read(dataProduct); + assertThat(dc.getComponent("a").readSamples()).containsExactly(1, 2, 3); + assertThat(dc.getComponent("b").readSamples()).containsExactly(4, 5, 6); + assertThat(dc.getComponent("c").readDistribution().getDistribution().internalType()) + .isEqualTo(DistributionType.categorical); + assertThat(dc.getComponent("d").readEstimate()).isEqualTo(estimate); + assertThat(dc.getComponent("e").readStrings()).containsExactly("do", "re", "mi"); + assertThat(dc.getComponent("f").readStrings()).containsExactly("just the one"); + assertThat(dc.getComponent("g").readBools()).containsExactly(true); + assertThat(dc.getComponent("h").readNumbers()).containsExactly(1.0, 1.5, 12345.67); + } + String hash = "68608005664459de456d75941430e7290031ddb5"; + check_last_coderun( + Arrays.asList( + new Triplet<>(dataProduct, "a", hash), + new Triplet<>(dataProduct, "b", hash), + new Triplet<>(dataProduct, "c", hash), + new Triplet<>(dataProduct, "d", hash), + new Triplet<>(dataProduct, "e", hash), + new Triplet<>(dataProduct, "f", hash), + new Triplet<>(dataProduct, "g", hash), + new Triplet<>(dataProduct, "h", hash)), + null); + } + + @Test + @Order(18) void testWriteGlobDP() { String dataProduct = "animal/dog"; String component = "example-samples"; @@ -494,7 +642,7 @@ void testWriteGlobDP() { } @Test - @Order(12) + @Order(19) void testReadNoGlob() { String dataProduct = "animal/dog"; String component = "example-samples"; @@ -508,7 +656,7 @@ void testReadNoGlob() { } @Test - @Order(13) + @Order(20) void testWriteGlobMultiDP() { String dataProduct1 = "animal/horse"; String dataProduct2 = "animal/mouse"; @@ -568,7 +716,7 @@ void testIssueWithFileObjects() { } @Test - @Order(14) + @Order(21) void testWriteSamplesMultipleComponentsAndIssues() { String dataProduct = "animal/dodo"; String component1 = "example-samples-dodo1"; @@ -600,7 +748,7 @@ void testWriteSamplesMultipleComponentsAndIssues() { } @Test - @Order(15) + @Order(22) void testReadSamplesMultipleComponentsAndIssues() { String dataProduct = "animal/dodo"; String component1 = "example-samples-dodo1"; @@ -627,7 +775,7 @@ void testReadSamplesMultipleComponentsAndIssues() { } @Test - @Order(16) + @Order(23) void testRead_oneIssueToMultipleComp_and_script() { String dataProduct = "animal/dodo"; String component1 = "example-samples-dodo1"; @@ -658,7 +806,7 @@ void testRead_oneIssueToMultipleComp_and_script() { } @Test - @Order(17) + @Order(24) void testCSV_writeLink() throws IOException { String dataProduct = "animal/ant"; try (var coderun = new Coderun(configPath, scriptPath, token)) { @@ -688,7 +836,7 @@ private List getRecordFromLine(String line) { } @Test - @Order(18) + @Order(25) void testCSV_readLink() throws IOException { String dataProduct = "animal/ant"; try (var coderun = new Coderun(configPath, scriptPath, token)) { @@ -707,7 +855,7 @@ void testCSV_readLink() throws IOException { } @Test - @Order(19) + @Order(26) void testCSV_writeLink_withIssue() throws IOException { String dataProduct = "animal/monkey"; String issue = "this does not seem to contain anything monkey-related"; @@ -726,7 +874,7 @@ void testCSV_writeLink_withIssue() throws IOException { } @Test - @Order(20) + @Order(27) void testCSV_readLink_withIssue() throws IOException { String dataProduct = "animal/ant"; String issue = "not enough orange"; @@ -748,7 +896,7 @@ void testCSV_readLink_withIssue() throws IOException { } @Test - @Order(21) + @Order(28) void testRewriteDPname() { String dataProduct = "animal/canine"; String component = "NumberOfLegs"; @@ -764,7 +912,7 @@ void testRewriteDPname() { } @Test - @Order(22) + @Order(29) void testReadRewrittenDPname() { String dataProduct = "animal/canine"; String component = "NumberOfLegs"; @@ -779,7 +927,7 @@ void testReadRewrittenDPname() { } @Test - @Order(23) + @Order(30) void testAltNS() { String dataProduct = "test/altns"; String component = "altNScompo"; @@ -794,7 +942,7 @@ void testAltNS() { } @Test - @Order(24) + @Order(31) void testAltNSread() { String dataProduct = "test/altns"; String component = "altNScompo"; @@ -808,7 +956,7 @@ void testAltNSread() { } @Test - @Order(25) + @Order(32) void testConfigFiletype() { String dataProduct = "animal/chicken"; try (var coderun = new Coderun(configPath, scriptPath, token)) { @@ -845,7 +993,7 @@ void testConfigFiletype() { @SuppressWarnings("EmptyTryBlock") @Test - @Order(26) + @Order(33) void emptyCoderun() { try (var coderun = new Coderun(configPath, scriptPath, token)) { // do nothing diff --git a/api/src/test/java/org/fairdatapipeline/api/testStorage_root.java b/api/src/test/java/org/fairdatapipeline/api/testStorage_root.java new file mode 100644 index 0000000..5fc4c20 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/api/testStorage_root.java @@ -0,0 +1,63 @@ +package org.fairdatapipeline.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +class testStorage_root { + String[] bad_remote_repos = + new String[] { + "host.xz:path/to/repo.git", + "host.xz:/path/to/repo.git/", + "/path/to/repo.git/", + "path/to/repo.git/", + "~/path/to/repo.git", + "host.xz:~user/path/to/repo.git/" + }; + String[] possible_remote_repos = + new String[] { + "ssh://user@host.xz:82/path/to/repo.git/", + "ssh://user@host.xz/path/to/repo.git/", + "ssh://host.xz:82/path/to/repo.git/", + "ssh://host.xz/path/to/repo.git/", + "ssh://user@host.xz/path/to/repo.git/", + "ssh://host.xz/path/to/repo.git/", + "ssh://user@host.xz/~user/path/to/repo.git/", + "ssh://host.xz/~user/path/to/repo.git/", + "ssh://user@host.xz/~/path/to/repo.git", + "ssh://host.xz/~/path/to/repo.git", + "user@host.xz:/path/to/repo.git/", + "user@host.xz:~user/path/to/repo.git/", + "user@host.xz:path/to/repo.git", + "rsync://host.xz/path/to/repo.git/", + "git://host.xz/path/to/repo.git/", + "git://host.xz/~user/path/to/repo.git/", + "http://host.xz/path/to/repo.git/", + "https://host.xz/path/to/repo.git/", + "file:///path/to/repo.git/", + "file://~/path/to/repo.git/" + }; + + @Test + void test_good_repo_strings() { + Arrays.stream(possible_remote_repos) + .forEach( + s -> { + String[] r = Storage_root.gitrepo_to_root(s); + assertThat(r).hasSize(2); + assertThat(r[0]).doesNotContain("null"); + assertThat(r[1]).doesNotContain("null"); + }); + } + + @Test + void test_bad_repo_strings() { + Arrays.stream(bad_remote_repos) + .forEach( + s -> { + String[] r = Storage_root.gitrepo_to_root(s); + assertThat(r).isEmpty(); + }); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/parameters/BoolListTest.java b/api/src/test/java/org/fairdatapipeline/parameters/BoolListTest.java new file mode 100644 index 0000000..c532ee0 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/parameters/BoolListTest.java @@ -0,0 +1,13 @@ +package org.fairdatapipeline.parameters; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class BoolListTest { + @Test + void makeBoolList() { + var boollist = ImmutableBoolList.builder().addBools(true, false, true).build(); + assertThat(boollist.bools()).contains(true, false); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/parameters/ComponentsDeserializerTest.java b/api/src/test/java/org/fairdatapipeline/parameters/ComponentsDeserializerTest.java index f356b8a..84da027 100644 --- a/api/src/test/java/org/fairdatapipeline/parameters/ComponentsDeserializerTest.java +++ b/api/src/test/java/org/fairdatapipeline/parameters/ComponentsDeserializerTest.java @@ -6,13 +6,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Map; import org.apache.commons.math3.random.RandomGenerator; import org.fairdatapipeline.distribution.Distribution.DistributionType; import org.fairdatapipeline.distribution.ImmutableDistribution; -import org.fairdatapipeline.estimate.ImmutableEstimate; import org.fairdatapipeline.mapper.DataPipelineMapper; -import org.fairdatapipeline.samples.ImmutableSamples; import org.junit.jupiter.api.*; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -25,17 +22,9 @@ class ComponentsDeserializerTest { + " \"shape\": 1,\n" + " \"type\": \"distribution\"\n" + " },\n" - + " \"example-estimate\": {\n" - + " \"type\": \"point-estimate\",\n" - + " \"value\": 1.0\n" - + " },\n" - + " \"example-samples\": {\n" - + " \"samples\": [\n" - + " 1,\n" - + " 2,\n" - + " 3\n" - + " ],\n" - + " \"type\": \"samples\"\n" + + " \"example-strings\": {\n" + + " \"strings\": [\"bram\",\"rosalie\"],\n" + + " \"type\": \"strings\"\n" + " }\n" + "}"; @@ -53,7 +42,6 @@ public void setUp() { void deserialize() throws JsonProcessingException { Components actualComponents = objectMapper.readValue(json, Components.class); - var estimate = ImmutableEstimate.builder().internalValue(1.0).rng(rng).build(); var distribution = ImmutableDistribution.builder() .internalType(DistributionType.gamma) @@ -61,19 +49,21 @@ void deserialize() throws JsonProcessingException { .internalScale(2) .rng(rng) .build(); - var samples = ImmutableSamples.builder().addSamples(1, 2, 3).rng(rng).build(); - var expectedComponents = + var strings = ImmutableStringList.builder().addStrings("bram", "rosalie").build(); + Components components = ImmutableComponents.builder() - .components( - Map.of( - "example-estimate", - estimate, - "example-distribution", - distribution, - "example-samples", - samples)) + .putComponents("example-distribution", distribution) + .putComponents("example-strings", strings) .build(); + /* var expectedComponents = + ImmutableComponents.builder() + .components( + Map.of( + "example-distribution", + distribution + )) + .build();*/ - assertThat(actualComponents).isEqualTo(expectedComponents); + assertThat(actualComponents).isEqualTo(components); } } diff --git a/api/src/test/java/org/fairdatapipeline/parameters/ComponentsSerializerTest.java b/api/src/test/java/org/fairdatapipeline/parameters/ComponentsSerializerTest.java index bc93fdb..e04c9b1 100644 --- a/api/src/test/java/org/fairdatapipeline/parameters/ComponentsSerializerTest.java +++ b/api/src/test/java/org/fairdatapipeline/parameters/ComponentsSerializerTest.java @@ -11,9 +11,7 @@ import org.apache.commons.math3.random.RandomGenerator; import org.fairdatapipeline.distribution.Distribution.DistributionType; import org.fairdatapipeline.distribution.ImmutableDistribution; -import org.fairdatapipeline.estimate.ImmutableEstimate; import org.fairdatapipeline.mapper.DataPipelineMapper; -import org.fairdatapipeline.samples.ImmutableSamples; import org.json.JSONException; import org.junit.jupiter.api.*; @@ -27,17 +25,9 @@ class ComponentsSerializerTest { + " \"shape\": 1,\n" + " \"type\": \"distribution\"\n" + " },\n" - + " \"example-estimate\": {\n" - + " \"type\": \"point-estimate\",\n" - + " \"value\": 1.0\n" - + " },\n" - + " \"example-samples\": {\n" - + " \"samples\": [\n" - + " 1,\n" - + " 2,\n" - + " 3\n" - + " ],\n" - + " \"type\": \"samples\"\n" + + " \"example-strings\": {\n" + + " \"strings\": [\"bram\",\"rosalie\"],\n" + + " \"type\": \"strings\"\n" + " }\n" + "}"; @@ -54,7 +44,6 @@ public void setUp() { @Test void serialize() throws IOException, JSONException { var writer = new StringWriter(); - var estimate = ImmutableEstimate.builder().internalValue(1.0).rng(rng).build(); var distribution = ImmutableDistribution.builder() .internalType(DistributionType.gamma) @@ -62,12 +51,11 @@ void serialize() throws IOException, JSONException { .internalScale(2) .rng(rng) .build(); - var samples = ImmutableSamples.builder().addSamples(1, 2, 3).rng(rng).build(); + var strings = ImmutableStringList.builder().addStrings("bram", "rosalie").build(); Components components = ImmutableComponents.builder() - .putComponents("example-estimate", estimate) .putComponents("example-distribution", distribution) - .putComponents("example-samples", samples) + .putComponents("example-strings", strings) .build(); objectMapper.writeValue(writer, components); diff --git a/api/src/test/java/org/fairdatapipeline/parameters/NumberListTest.java b/api/src/test/java/org/fairdatapipeline/parameters/NumberListTest.java new file mode 100644 index 0000000..ed9c549 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/parameters/NumberListTest.java @@ -0,0 +1,25 @@ +package org.fairdatapipeline.parameters; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class NumberListTest { + + /** + * test that a mix of ints and doubles will end up as all doubles. (necessary for TOML - it can't + * read a 'heterogenous array' of mixed int & double numbers) + */ + @Test + void makeNumberList() { + var numberlist = ImmutableNumberList.builder().addNumbers(1, 1.5, 12345.67).build(); + assertThat(numberlist.numbers()).contains(1.0, 1.5, 12345.67); + } + + /** given just ints, they should NOT be converted to doubles. */ + @Test + void makeNumberList2() { + var numberlist = ImmutableNumberList.builder().addNumbers(1, 2, 12).build(); + assertThat(numberlist.numbers()).contains(1, 2, 12); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/parameters/ParameterDataReaderTest.java b/api/src/test/java/org/fairdatapipeline/parameters/ParameterDataReaderTest.java index ac4cc10..12b13ea 100644 --- a/api/src/test/java/org/fairdatapipeline/parameters/ParameterDataReaderTest.java +++ b/api/src/test/java/org/fairdatapipeline/parameters/ParameterDataReaderTest.java @@ -18,12 +18,14 @@ class ParameterDataReaderTest { private CleanableFileChannel fileChannel; private TomlReader tomlReader; private Estimate mockEstimate; + private StringList mockStringList; @BeforeAll public void setUp() { this.fileChannel = mock(CleanableFileChannel.class); this.tomlReader = mock(TomlReader.class); this.mockEstimate = mock(Estimate.class); + this.mockStringList = mock(StringList.class); } @Test diff --git a/api/src/test/java/org/fairdatapipeline/parameters/StringListTest.java b/api/src/test/java/org/fairdatapipeline/parameters/StringListTest.java new file mode 100644 index 0000000..a230c99 --- /dev/null +++ b/api/src/test/java/org/fairdatapipeline/parameters/StringListTest.java @@ -0,0 +1,13 @@ +package org.fairdatapipeline.parameters; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class StringListTest { + @Test + void makeStringList() { + var stringlist = ImmutableStringList.builder().addStrings("bla", "ble").build(); + assertThat(stringlist.strings()).contains("bla", "ble"); + } +} diff --git a/api/src/test/java/org/fairdatapipeline/samples/SamplesTest.java b/api/src/test/java/org/fairdatapipeline/samples/SamplesTest.java index 87ea208..77d329b 100644 --- a/api/src/test/java/org/fairdatapipeline/samples/SamplesTest.java +++ b/api/src/test/java/org/fairdatapipeline/samples/SamplesTest.java @@ -39,6 +39,12 @@ void derivedSamplesFromSamples() { assertThat(samples.getSamples()).containsExactly(1, 2, 3); } + @Test + void heterogenousToDoubles() { + var samples = ImmutableSamples.builder().addSamples(1.4, 2.5, 3).rng(rng).build(); + assertThat(samples.getSamples()).containsExactly(1.4, 2.5, 3.0); + } + @Test void derivedDistributionFromSamples() { var samples = ImmutableSamples.builder().addSamples(1, 2, 3).rng(rng).build(); diff --git a/api/src/test/java/org/fairdatapipeline/toml/TomlReaderPairwiseIntegrationTest.java b/api/src/test/java/org/fairdatapipeline/toml/TomlReaderPairwiseIntegrationTest.java index 141fdc2..b75c77d 100644 --- a/api/src/test/java/org/fairdatapipeline/toml/TomlReaderPairwiseIntegrationTest.java +++ b/api/src/test/java/org/fairdatapipeline/toml/TomlReaderPairwiseIntegrationTest.java @@ -7,9 +7,13 @@ import com.fasterxml.jackson.core.type.TypeReference; import java.io.StringReader; import org.apache.commons.math3.random.RandomGenerator; +import org.fairdatapipeline.distribution.Distribution; +import org.fairdatapipeline.distribution.ImmutableDistribution; import org.fairdatapipeline.estimate.ImmutableEstimate; import org.fairdatapipeline.parameters.Components; import org.fairdatapipeline.parameters.ImmutableComponents; +import org.fairdatapipeline.parameters.ImmutableStringList; +import org.fairdatapipeline.samples.ImmutableSamples; import org.junit.jupiter.api.*; @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -17,6 +21,22 @@ class TomlReaderPairwiseIntegrationTest { private final String toml = "[example-estimate]\n" + "type = \"point-estimate\"\n" + "value = 1.0"; + private final String toml2 = + "[example-estimate]\n" + + "value = 1.0\n" + + "type = \"point-estimate\"\n" + + "[example-distribution]\n" + + "distribution = \"gamma\"\n" + + "shape = 1.0\n" + + "scale = 2.0\n" + + "type = \"distribution\"\n" + + "[example-samples]\n" + + "samples = [1.5, 2.0, 3.0]\n" + + "type = \"samples\"\n" + + "[example-strings]\n" + + "strings = [\"bla\", \"blo\"]\n" + + "type = \"strings\"\n"; + private RandomGenerator rng; @BeforeAll @@ -36,4 +56,31 @@ void read() { assertThat(components_read).isEqualTo(components); } + + @Test + void read2() { + TomlReader tomlReader = new TomlReader(new TOMLMapper(rng)); + var reader = new StringReader(toml2); + var estimate = ImmutableEstimate.builder().internalValue(1.0).rng(rng).build(); + Components components = + ImmutableComponents.builder() + .putComponents("example-estimate", estimate) + .putComponents( + "example-distribution", + ImmutableDistribution.builder() + .internalShape(1) + .internalScale(2) + .internalType(Distribution.DistributionType.gamma) + .rng(rng) + .build()) + .putComponents( + "example-samples", + ImmutableSamples.builder().addSamples(1.5, 2, 3).rng(rng).build()) + .putComponents( + "example-strings", ImmutableStringList.builder().addStrings("bla", "blo").build()) + .build(); + Components components_read = tomlReader.read(reader, new TypeReference<>() {}); + + assertThat(components_read.components()).containsAllEntriesOf(components.components()); + } } diff --git a/api/src/test/java/org/fairdatapipeline/toml/TomlWriterPairwiseIntegrationTest.java b/api/src/test/java/org/fairdatapipeline/toml/TomlWriterPairwiseIntegrationTest.java index 8ee798d..d0aa8cd 100644 --- a/api/src/test/java/org/fairdatapipeline/toml/TomlWriterPairwiseIntegrationTest.java +++ b/api/src/test/java/org/fairdatapipeline/toml/TomlWriterPairwiseIntegrationTest.java @@ -3,7 +3,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.skyscreamer.jsonassert.JSONAssert.assertEquals; import java.io.StringWriter; import org.apache.commons.math3.random.RandomGenerator; @@ -12,26 +11,28 @@ import org.fairdatapipeline.estimate.ImmutableEstimate; import org.fairdatapipeline.parameters.Components; import org.fairdatapipeline.parameters.ImmutableComponents; +import org.fairdatapipeline.parameters.ImmutableStringList; import org.fairdatapipeline.samples.ImmutableSamples; -import org.json.JSONException; import org.junit.jupiter.api.*; @TestInstance(TestInstance.Lifecycle.PER_CLASS) class TomlWriterPairwiseIntegrationTest { + private final String expectedToml = - "[\"example estimate\"]\n" - + "type = \"point-estimate\"\n" + "[example-estimate]\n" + "value = 1.0\n" - + "\n" + + "type = \"point-estimate\"\n" + "[example-distribution]\n" - + "type = \"distribution\"\n" + "distribution = \"gamma\"\n" - + "shape = 1\n" - + "scale = 2\n" - + "\n" + + "shape = 1.0\n" + + "scale = 2.0\n" + + "type = \"distribution\"\n" + "[example-samples]\n" + + "samples = [1.5, 2.0, 3.0]\n" + "type = \"samples\"\n" - + "samples = [ 1, 2, 3,]"; + + "[example-strings]\n" + + "strings = [\"bla\", \"blo\"]\n" + + "type = \"strings\"\n"; private TOMLMapper tomlMapper; private RandomGenerator rng; @@ -44,7 +45,7 @@ public void setUp() { } @Test - void write() throws JSONException { + void write() { var estimate = ImmutableEstimate.builder().internalValue(1.0).rng(rng).build(); var distribution = ImmutableDistribution.builder() @@ -53,13 +54,15 @@ void write() throws JSONException { .internalScale(2) .rng(rng) .build(); - var samples = ImmutableSamples.builder().addSamples(1, 2, 3).rng(rng).build(); + var samples = ImmutableSamples.builder().addSamples(1.5, 2, 3).rng(rng).build(); + var strings = ImmutableStringList.builder().addStrings("bla", "blo").build(); Components components = ImmutableComponents.builder() - .putComponents("example estimate", estimate) + .putComponents("example-estimate", estimate) .putComponents("example-distribution", distribution) .putComponents("example-samples", samples) + .putComponents("example-strings", strings) .build(); var writer = new StringWriter(); @@ -68,7 +71,6 @@ void write() throws JSONException { tomlWriter.write(writer, components); var actualToml = writer.toString(); - assertThat(actualToml).isNotBlank(); - assertEquals(actualToml, expectedToml, true); + assertThat(actualToml).isEqualTo(expectedToml); } } diff --git a/api/src/test/resources/config-stdapi.yaml b/api/src/test/resources/config-stdapi.yaml index bcddb5f..b768141 100644 --- a/api/src/test/resources/config-stdapi.yaml +++ b/api/src/test/resources/config-stdapi.yaml @@ -25,6 +25,9 @@ read: - data_product: human/multicomp use: version: 0.0.1 + - data_product: human/allsortscomp + use: + version: 0.0.1 - data_product: animal/dog use: version: 0.0.1 @@ -50,6 +53,10 @@ read: version: 0.0.1 write: + - data_product: human/population2 + description: Coderun Integration test + use: + version: 0.0.1 - data_product: human/population description: Coderun Integration test use: @@ -58,6 +65,10 @@ write: description: Coderun Integration test for distribution use: version: 0.0.1 + - data_product: human/cdistribution2 + description: Coderun Integration test for cdistribution + use: + version: 0.0.1 - data_product: human/cdistribution description: Coderun Integration test for cdistribution use: @@ -66,10 +77,18 @@ write: description: Coderun Integration test for samples use: version: 0.0.1 + - data_product: human/samples2 + description: Coderun Integration test for samples + use: + version: 0.0.1 - data_product: human/multicomp description: Coderun Integration test for samples multiple components use: version: 0.0.1 + - data_product: human/allsortscomp + description: Coderun Integration test for writing a whole variety of components + use: + version: 0.0.1 - data_product: animal/* description: Coderun Integration test starglob animal use: diff --git a/api/src/test/resources/config_remrepo1.yaml b/api/src/test/resources/config_remrepo1.yaml new file mode 100644 index 0000000..743df17 --- /dev/null +++ b/api/src/test/resources/config_remrepo1.yaml @@ -0,0 +1,21 @@ +fail_on_hash_mismatch: True +run_metadata: + description: Coderun Integration test + local_data_registry_url: http://localhost:8000/api/ + default_input_namespace: CoderunTest2 + default_output_namespace: CoderunTest2 + local_repo: local repo + script: '' + latest_commit: 988637ffb95381375f6b22e2281fea1bcb6c8fe0 + remote_repo: user@host.xz:/path/to/repo.git + +read: + - data_product: human/population + use: + version: 0.0.1 + +write: + - data_product: human/population + description: Coderun Integration test + use: + version: 0.0.1 \ No newline at end of file