diff --git a/benchmarks/matbench_v0.1_StackNet-mat_v0.2/ga_stacknet.py b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/ga_stacknet.py new file mode 100644 index 00000000..470a2a77 --- /dev/null +++ b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/ga_stacknet.py @@ -0,0 +1,481 @@ +# train autoencoder for regression with no compression in the bottleneck layer +import pandas as pd +import numpy as np + +from sklearn.datasets import make_regression +from sklearn.preprocessing import MinMaxScaler +from sklearn.model_selection import train_test_split +import matplotlib.pyplot as plt +import seaborn as sns +import warnings +import xgboost as xgb + +# baseline in performance with support vector regression model +from sklearn.datasets import make_regression +from sklearn.preprocessing import MinMaxScaler +from sklearn.model_selection import train_test_split +from sklearn.svm import SVR +from sklearn.metrics import mean_absolute_error +from sklearn.metrics import r2_score +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.ensemble import AdaBoostRegressor +from sklearn.ensemble import GradientBoostingRegressor +from sklearn.ensemble import RandomForestRegressor + +from sklearn.preprocessing import RobustScaler +from sklearn.ensemble import RandomForestRegressor +from sklearn.linear_model import LinearRegression +from sklearn.linear_model import LogisticRegression +from sklearn.linear_model import Ridge +from xgboost import XGBRegressor +from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error +from sklearn.pipeline import Pipeline +from sklearn.kernel_ridge import KernelRidge +from lightgbm import LGBMRegressor +from sklearn.linear_model import BayesianRidge + +from sklearn.metrics import r2_score +from sklearn.metrics import mean_absolute_error + + +from sklearn.model_selection import cross_val_score +from sklearn.metrics import make_scorer +from sklearn.model_selection import KFold +from sklearn.model_selection import cross_validate +from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error, mean_absolute_percentage_error, explained_variance_score, mean_pinball_loss + + +import sys +import pickle +import joblib +import re +import json + + +import pygad + +import gzip +import shutil + +import os + + + +from matbench.bench import MatbenchBenchmark + +from pystacknet.pystacknet import StackNetRegressor +from matminer.datasets import load_dataset +from matminer.featurizers.composition.element import ElementFraction +from matminer.featurizers.composition.element import TMetalFraction +from matminer.featurizers.composition.element import Stoichiometry + +from matminer.featurizers.composition.composite import Meredig +from matminer.featurizers.composition.element import BandCenter + +from matminer.featurizers.conversions import StrToComposition + +from matminer.datasets import get_all_dataset_info + +import warnings +warnings.filterwarnings("ignore") + +def model_test(models, model_names, X_train, y_train, X_test, y_test, norm=True): + #utilizing for-loop to quickly analyze all 5 models + mae = {} + rmse = {} + r2 = {} + for model, name in zip(models, model_names): + if norm: + model = Pipeline([('scaler', RobustScaler()), ('model', model)]) + else: + model = Pipeline([('model', model)]) + model.fit(X_train, y_train) + y_pred = model.predict(X_test) + + mae[name] = mean_absolute_error(y_test, y_pred) + rmse[name] = mean_squared_error(y_test, y_pred, squared=False) + r2[name] = r2_score(y_test, y_pred) + # print(f'{name} \n R-Squared: {r2:.5f} \n MAE: {mae:.5f} \n RMSE: {rmse:.5f}') + return mae, rmse, r2 + + +def cross_val_test(model, dataX, dataY, model_type='regression', n_cv=10): + + # Розбиття для подальшої крос-валідації + cv = KFold(n_splits=n_cv, shuffle=True) + # Визначення набору метрик + if model_type == 'regression': + scorer = {'r2':make_scorer(r2_score), + 'mae': make_scorer(mean_absolute_error), + 'mse': make_scorer(mean_squared_error), + 'mape': make_scorer(mean_absolute_percentage_error)} + if model_type == 'classification': + pass + + # Оцінка якості моделі на різних наборах даних + scores = cross_validate(model, dataX, dataY, scoring=scorer, cv=cv, return_train_score=True) + return scores + +def remove_zero_lists(list_of_lists): + """Видаляє всі списки, що складаються виключно з нулів, зі списку списків.""" + return [sublist for sublist in list_of_lists if not all(x == 0 for x in sublist)] + + +def split_list(input_list, chunk_size): + """Розділити список на підсписки заданої довжини.""" + return [input_list[i:i + chunk_size] for i in range(0, len(input_list), chunk_size)] + +def find_indices_of_ones(binary_list): + """Повертає список індексів, на позиціях яких у бінарному списку стоять одиниці.""" + return [index for index, value in enumerate(binary_list) if value == 1] + +def select_models_by_indices(ind, models): + """Генерує список підсписків, де кожен підсписок відповідає індексам з `ind` для списку `models`.""" + return [[models[index] for index in sublist] for sublist in ind] + +# # sol - приклад хромосоми, з якої генерується модель +# sol = [1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + +# split_list(sol, 7) + +# gen_stacknet(sol) + + + +def plot_scatter(y_true, y_pred): + """ + Функція для побудови точкової діаграми справжніх значень та прогнозованих значень. + + :param y_true: список або масив справжніх значень + :param y_pred: список або масив прогнозованих значень + """ + plt.figure(figsize=(8, 6)) + plt.scatter(y_true, y_pred, color='blue', marker='o') + plt.plot([min(y_true), max(y_true)], [min(y_true), max(y_true)], color='red', linestyle='--') # Лінія ідеального прогнозу + plt.title('Точкова діаграма: справжні значення vs. прогнозовані значення') + plt.xlabel('Тестові значення (y_test)') + plt.ylabel('Прогнозовані значення (y_pred)') + plt.grid(True) + plt.show() + + +# sys.path.insert(0,'../..') + +def unzip(path_to_gz_file, output_path): +# Відкриваємо gz файл + with gzip.open(path_to_gz_file, 'rb') as f_in: + # Відкриваємо вихідний файл для запису + with open(output_path, 'wb') as f_out: + # Копіюємо вміст gz файлу в вихідний файл + shutil.copyfileobj(f_in, f_out) + + print("Файл успішно розархівовано.") + + + +def delete_file(file_path): + """Видаляє файл, якщо він існує.""" + try: + os.remove(file_path) + print(f"Файл {file_path} успішно видалено.") + except FileNotFoundError: + print(f"Файл {file_path} не знайдено.") + except PermissionError: + print(f"Недостатньо прав для видалення файлу {file_path}.") + except Exception as e: + print(f"Помилка при видаленні файлу: {e}") + + + +def process_json_files(directory): + + unzip('./results.json.gz', './results.json') + results = [] + + for filename in os.listdir(directory): + if filename.endswith('.json'): + filepath = os.path.join(directory, filename) + with open(filepath, 'r') as file: + data = json.load(file) + + # Extracting the MAE scores from each fold + mae_scores = [] + for fold in range(5): + key = f"fold_{fold}" + mae_scores.append(data["tasks"]["matbench_steels"]["results"][key]["scores"]["mae"]) + + # Calculating mean and standard deviation + mae_mean = np.mean(mae_scores) + mae_std = np.std(mae_scores) + + results.append({ + "filename": filename, + "mae_scores": mae_scores, + "mean": mae_mean, + "std": mae_std + }) + + # delete_file('./results.json') + # delete_file('./results.json.gz') + + return results + + +def data_preproc(df_steels, compos_name): + + + stc = StrToComposition() + df_steels = stc.featurize_dataframe(df_steels, compos_name, pbar=False) + + ef = ElementFraction() + tm = TMetalFraction() + st = Stoichiometry() + meredig = Meredig() + bc = BandCenter() + + df_steels_bc = bc.featurize_dataframe(df_steels, "composition") + df_steels_ef = ef.featurize_dataframe(df_steels, "composition") + + df_steels_ef = df_steels_ef.loc[:, (df_steels_ef == 0).mean() <= 0.6] + df_steels_ef + + df_steels_tm = tm.featurize_dataframe(df_steels, "composition") + + df_steels_st = st.featurize_dataframe(df_steels, "composition") + + # df_steels_meredig = meredig.featurize_dataframe(df_steels, "composition") + + data_steel = df_steels_ef.drop([compos_name, 'composition'], axis=1) + + df = pd.concat([data_steel, + df_steels_st['0-norm'], df_steels_st['2-norm'], + df_steels_st['3-norm'], df_steels_st['5-norm'], + df_steels_st['7-norm'], df_steels_st['10-norm'], + df_steels_tm['transition metal fraction'], + df_steels_bc['band center'] + ], axis=1) + + return df + +def gen_stacknet(sol): + + models = [RandomForestRegressor(random_state=42), + LinearRegression(), + Ridge(random_state=42), + XGBRegressor(random_state=42), + HistGradientBoostingRegressor(random_state=42), + AdaBoostRegressor(random_state=42), + GradientBoostingRegressor(learning_rate=0.01, n_estimators=1000, + subsample=1.0, criterion='friedman_mse', + min_samples_split=2, min_samples_leaf=1, + min_weight_fraction_leaf=0.0, max_depth=5, random_state=42), + + ] + + len_model = len(models) + restacking = bool(sol[-2]) + retraining = bool(sol[-1]) + layers = sol[0:-2] + sublayers = split_list(layers, len_model) + sublayers = remove_zero_lists(sublayers) + if sublayers: + models_by_layers = [find_indices_of_ones(sublayers[i]) for i in range(len(sublayers))] + selected_models = select_models_by_indices(models_by_layers, models) + selected_models.append([selected_models[-1][-1]]) + + else: + selected_models = models[0] + + stack_net = StackNetRegressor(selected_models, metric="r2", folds=5, + restacking=restacking, use_retraining=retraining, + random_state=42, n_jobs=16, verbose=0) + + return stack_net + + +def model_eval(model): + + mb = MatbenchBenchmark(autoload=False, subset=['matbench_steels']) + + for task in mb.tasks: + task.load() + for fold in task.folds: + + # Inputs are either chemical compositions as strings + # or crystal structures as pymatgen.Structure objects. + # Outputs are either floats (regression tasks) or bools (classification tasks) + train_inputs, train_outputs = task.get_train_and_val_data(fold) + + df_steels_mb = pd.DataFrame({'compos':list(train_inputs), 'target':list(train_outputs)}) + + df = data_preproc(df_steels_mb, 'compos') + + X_train = df.drop('target', axis=1) + y_train = df['target'] + + + # train and validate your model + model.fit(X_train,y_train) + + # Get testing data + test_inputs = task.get_test_data(fold, include_target=False) + + test_df = pd.DataFrame({'compos':list(test_inputs)}) + X_test = data_preproc(test_df, 'compos') + + # Predict on the testing data + # Your output should be a pandas series, numpy array, or python iterable + # where the array elements are floats or bools + # predictions = my_model.predict(test_inputs) + predictions = model.predict(X_test) + + predictions = predictions.flatten() + + # Record your data! + task.record(fold, predictions) + + + + + # Save your results + mb.to_file("results.json.gz") + + return 0 + + + + +def fitness_eval(results_dic, lambd=0.0): + + fitness = 1 / (results_dic['mean'] + lambd*results_dic['std'] ) * 1000 + return fitness + + +sol_dict = [] + +def fitness_func(ga_instance, solution, solution_idx): + global sol_dict + model = gen_stacknet(solution) + model_eval(model) + directory = "./" # Replace with the path to your directory with json files + results = process_json_files(directory) + fitness = fitness_eval(results[0]) + print('------------sol----------\n', solution) + print('------------sol_idx----------\n', solution_idx) + print('------------fit----------\n', fitness) + sol_dict.append([solution, solution_idx, fitness, results[0]['mean'], results[0]['std']]) + return fitness + + + +# 5 layers for initial solution +# init_solution = [0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0] + +# 5 layers for initial solution +# len([0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,1,0]) +# init_solution = [1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + +init_solution = [1,0,1,0,0,1,1,0,0,1,0,0,1,1,0,1,0,1,0,1,1,1,0] + + +def genetic_instance(): + global init_population + global init_solution + ga_instance = pygad.GA(num_generations=10, + num_parents_mating=2, + fitness_func=fitness_func, + initial_population=init_population, + # sol_per_pop=int(sol_per_pop), + num_genes=len(init_solution), + # on_start=on_start, + # on_fitness=on_fitness, + # on_parents=on_parents, + # on_crossover=on_crossover, + # on_mutation=on_mutation, + # on_generation=on_generation, + crossover_probability= 60/100 + 0.1, + mutation_probability= 80/100 + 0.1, # [0,1] + mutation_by_replacement=False, # True + mutation_percent_genes = 80, # [0,100] + # on_stop=on_stop + save_solutions=True, + crossover_type='uniform', # 83.88 13 robust + mutation_type='swap', + + # crossover_type='scattered', + # mutation_type='swap', + + # crossover_type='uniform', # + # mutation_type='inversion', + ) + + ga_instance.run() + + return ga_instance + + +import random + +def generate_random_binary_list(length): + """Генерує випадковий бінарний список заданої довжини.""" + return [random.randint(0, 1) for _ in range(length)] + + +def set_init_population(n_pop): + global init_solution + + init_pop = [init_solution] + for i in range(n_pop): + new_sol = generate_random_binary_list(len(init_solution)) + init_pop.append(new_sol) + + # self.init_population = init_pop + return init_pop + + + +def ga_results(ga_instance): + ga_instance.plot_genes(color='blue') + ga_instance.plot_fitness(color='blue') + ga_instance.plot_new_solution_rate(color='blue') + + + +init_population = set_init_population(4) + + +ga = genetic_instance() + + + +solution, solution_fitness, solution_idx = ga.best_solution() +print("Parameters of the best solution : {solution}".format(solution=solution)) +print("Fitness value of the best solution = {solution_fitness}".format(solution_fitness=solution_fitness)) + +best_sol = solution + +model = gen_stacknet(best_sol) +model_eval(model) +directory = "./" # Replace with the path to your directory with json files +results = process_json_files(directory) + +ga_results(ga) + + +# len([0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,1,0]) +# ([0., 0., 1., 0., 1., 1., 1., 0., 1., 1., 0., 1., 1., 1., 0., 1., 0., +# 1., 0., 0., 1., 1., 0.]) 83.88 13 robust + +# [0., 0., 1., 0., 0., 0., 1., 0., 1., 0., 1., 1., 1., 1., 0., 1., 1., +# 0., 0., 1., 1., 1., 0.]) + + +model = gen_stacknet([0., 0., 1., 0., 1., 1., 1., 0., 1., 1., 0., 1., 1., 1., 0., 1., 0.,1., 0., 0., 1., 1., 0.]) + +model_eval(model) +directory = "./" # Replace with the path to your directory with json files +results = process_json_files(directory) + +# fitness_func(ga, [0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,1,1,1,0], 0) + + diff --git a/benchmarks/matbench_v0.1_StackNet-mat_v0.2/info.json b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/info.json new file mode 100644 index 00000000..debb16cc --- /dev/null +++ b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/info.json @@ -0,0 +1,8 @@ +{ + "authors": "Oleksii Kudin, Anastasiiya Krivokhata, Yuriy Belokon, Dmytro Kruglyak, Sergii Ilin", + "algorithm": "StackNet-mat v02", + "algorithm_long": "A method based on basic machine learning algorithms: RandomFores, HistGradientBoosting, AdaBoost, GradientBoosting, and the use of Stacknet as a metamodel", + "bibtex_refs": " ", + "notes": " ", + "requirements": {"python": ["scikit-learn==1.5.0", "numpy==1.25.2", "matbench==0.6", "matminer==0.9.2", "pystacknet==0.0.1", "tensorflow==2.15.0"]} +} diff --git a/benchmarks/matbench_v0.1_StackNet-mat_v0.2/requirements.txt b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/requirements.txt new file mode 100644 index 00000000..aae1ce09 --- /dev/null +++ b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/requirements.txt @@ -0,0 +1,507 @@ +absl-py==1.4.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +alabaster==0.7.16 +albumentations==1.3.1 +altair==4.2.2 +annotated-types==0.7.0 +anyio==3.7.1 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +array_record==0.5.1 +arviz==0.15.1 +astropy==5.3.4 +astunparse==1.6.3 +async-timeout==4.0.3 +atpublic==4.1.0 +attrs==23.2.0 +audioread==3.0.1 +autograd==1.6.2 +Babel==2.15.0 +backcall==0.2.0 +beautifulsoup4==4.12.3 +bidict==0.23.1 +bigframes==1.6.0 +black==22.3.0 +bleach==6.1.0 +blinker==1.4 +blis==0.7.11 +blosc2==2.0.0 +bokeh==3.3.4 +bqplot==0.12.43 +branca==0.7.2 +build==1.2.1 +CacheControl==0.14.0 +cachetools==5.3.3 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +chex==0.1.86 +click==8.1.7 +click-plugins==1.1.1 +cligj==0.7.2 +cloudpathlib==0.16.0 +cloudpickle==2.2.1 +cmake==3.27.9 +cmdstanpy==1.2.2 +colorcet==3.1.0 +colorlover==0.3.0 +colour==0.1.5 +community==1.0.0b1 +confection==0.1.4 +cons==0.4.6 +contextlib2==21.6.0 +contourpy==1.2.1 +coverage==6.4.3 +cryptography==42.0.7 +cuda-python==12.2.1 +cudf-cu12 @ https://pypi.nvidia.com/cudf-cu12/cudf_cu12-24.4.1-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=57366e7ef09dc63e0b389aff20df6c37d91e2790065861ee31a4720149f5b694 +cufflinks==0.17.3 +cupy-cuda12x==12.2.0 +cvxopt==1.3.2 +cvxpy==1.3.4 +cycler==0.12.1 +cymem==2.0.8 +Cython==3.0.10 +dask==2023.8.1 +datascience==0.17.6 +db-dtypes==1.2.0 +dbus-python==1.2.18 +debugpy==1.6.6 +decorator==4.4.2 +defusedxml==0.7.1 +distributed==2023.8.1 +distro==1.7.0 +dlib==19.24.4 +dm-tree==0.1.8 +dnspython==2.6.1 +docstring_parser==0.16 +docutils==0.18.1 +dopamine_rl==4.0.9 +duckdb==0.10.2 +earthengine-api==0.1.403 +easydict==1.13 +ecos==2.0.13 +editdistance==0.6.2 +eerepr==0.0.4 +en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl#sha256=86cc141f63942d4b2c5fcee06630fd6f904788d2f0ab005cce45aadb8fb73889 +entrypoints==0.4 +et-xmlfile==1.1.0 +etils==1.7.0 +etuples==0.3.9 +exceptiongroup==1.2.1 +fastai==2.7.15 +fastcore==1.5.38 +fastdownload==0.0.7 +fastjsonschema==2.19.1 +fastprogress==1.0.3 +fastrlock==0.8.2 +filelock==3.14.0 +fiona==1.9.6 +firebase-admin==5.3.0 +flake8==4.0.1 +Flask==2.2.5 +flatbuffers==24.3.25 +flax==0.8.3 +folium==0.14.0 +fonttools==4.51.0 +frozendict==2.4.4 +frozenlist==1.4.1 +fsspec==2023.6.0 +future==0.18.3 +gast==0.5.4 +gcsfs==2023.6.0 +GDAL==3.6.4 +gdown==5.1.0 +geemap==0.32.1 +gensim==4.3.2 +geocoder==1.38.1 +geographiclib==2.0 +geopandas==0.13.2 +geopy==2.3.0 +gin-config==0.5.0 +glob2==0.7 +google==2.0.3 +google-ai-generativelanguage==0.6.4 +google-api-core==2.11.1 +google-api-python-client==2.84.0 +google-auth==2.27.0 +google-auth-httplib2==0.1.1 +google-auth-oauthlib==1.2.0 +google-cloud-aiplatform==1.52.0 +google-cloud-bigquery==3.21.0 +google-cloud-bigquery-connection==1.12.1 +google-cloud-bigquery-storage==2.25.0 +google-cloud-core==2.3.3 +google-cloud-datastore==2.15.2 +google-cloud-firestore==2.11.1 +google-cloud-functions==1.13.3 +google-cloud-iam==2.15.0 +google-cloud-language==2.13.3 +google-cloud-resource-manager==1.12.3 +google-cloud-storage==2.8.0 +google-cloud-translate==3.11.3 +google-colab @ file:///colabtools/dist/google-colab-1.0.0.tar.gz#sha256=be2293f8ecde760dca63eb67279ba06283ad80f0e13e1c08343f5ef1b33c7116 +google-crc32c==1.5.0 +google-generativeai==0.5.4 +google-pasta==0.2.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.63.0 +googledrivedownloader==0.4 +graphviz==0.20.3 +greenlet==3.0.3 +grpc-google-iam-v1==0.13.0 +grpcio==1.64.0 +grpcio-status==1.48.2 +gspread==6.0.2 +gspread-dataframe==3.3.1 +gym==0.25.2 +gym-notices==0.0.8 +h5netcdf==1.3.0 +h5py==3.9.0 +holidays==0.49 +holoviews==1.17.1 +html5lib==1.1 +httpimport==1.3.1 +httplib2==0.22.0 +huggingface-hub==0.23.1 +humanize==4.7.0 +hyperopt==0.2.7 +ibis-framework==8.0.0 +idna==3.7 +imageio==2.31.6 +imageio-ffmpeg==0.4.9 +imagesize==1.4.1 +imbalanced-learn==0.10.1 +imgaug==0.4.0 +importlib_metadata==7.1.0 +importlib_resources==6.4.0 +imutils==0.5.4 +inflect==7.0.0 +iniconfig==2.0.0 +intel-openmp==2023.2.4 +ipyevents==2.0.2 +ipyfilechooser==0.6.0 +ipykernel==5.5.6 +ipyleaflet==0.18.2 +ipython==7.34.0 +ipython-genutils==0.2.0 +ipython-sql==0.5.0 +ipytree==0.2.2 +ipywidgets==7.7.1 +isort==5.10.1 +itsdangerous==2.2.0 +jax==0.4.26 +jaxlib @ https://storage.googleapis.com/jax-releases/cuda12/jaxlib-0.4.26+cuda12.cudnn89-cp310-cp310-manylinux2014_x86_64.whl#sha256=813cf1fe3e7ca4dbf5327d6e7b4fc8521e92d8bba073ee645ae0d5d036a25750 +jeepney==0.7.1 +jellyfish==1.0.3 +jieba==0.42.1 +Jinja2==3.1.4 +joblib==1.4.2 +jsonpickle==3.0.4 +jsonschema==4.19.2 +jsonschema-specifications==2023.12.1 +jupyter-client==6.1.12 +jupyter-console==6.1.0 +jupyter-server==1.24.0 +jupyter_core==5.7.2 +jupyterlab_pygments==0.3.0 +jupyterlab_widgets==3.0.10 +kaggle==1.6.14 +kagglehub==0.2.5 +keras==2.15.0 +keyring==23.5.0 +kiwisolver==1.4.5 +langcodes==3.4.0 +language_data==1.2.0 +latexcodec==3.0.0 +launchpadlib==1.10.16 +lazr.restfulclient==0.14.4 +lazr.uri==1.0.6 +lazy_loader==0.4 +libclang==18.1.1 +librosa==0.10.2.post1 +lightgbm==4.1.0 +linkify-it-py==2.0.3 +llvmlite==0.41.1 +locket==1.0.0 +logical-unification==0.4.6 +lxml==4.9.4 +malloy==2023.1067 +marisa-trie==1.1.1 +Markdown==3.6 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +-e git+https://github.com/hackingmaterials/matbench@936176db18ca4cd7b38cbd957c017a5bac770c6b#egg=matbench +matminer==0.9.2 +matplotlib==3.7.1 +matplotlib-inline==0.1.7 +matplotlib-venn==0.11.10 +mccabe==0.6.1 +mdit-py-plugins==0.4.1 +mdurl==0.1.2 +miniKanren==1.0.3 +missingno==0.5.2 +mistune==0.8.4 +mizani==0.9.3 +mkl==2023.2.0 +ml-dtypes==0.2.0 +mlxtend==0.22.0 +monty==2024.5.24 +more-itertools==10.1.0 +moviepy==1.0.3 +mpmath==1.3.0 +msgpack==1.0.8 +multidict==6.0.5 +multipledispatch==1.0.0 +multitasking==0.0.11 +murmurhash==1.0.10 +music21==9.1.0 +mypy-extensions==1.0.0 +natsort==8.4.0 +nbclassic==1.0.0 +nbclient==0.10.0 +nbconvert==6.5.4 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.3 +nibabel==4.0.2 +nltk==3.8.1 +notebook==6.5.5 +notebook_shim==0.2.4 +numba==0.58.1 +numexpr==2.10.0 +numpy==1.25.2 +nvtx==0.2.10 +oauth2client==4.1.3 +oauthlib==3.2.2 +opencv-contrib-python==4.8.0.76 +opencv-python==4.8.0.76 +opencv-python-headless==4.9.0.80 +openpyxl==3.1.2 +opt-einsum==3.3.0 +optax==0.2.2 +orbax-checkpoint==0.4.4 +osqp==0.6.2.post8 +packaging==24.0 +palettable==3.3.3 +pandas==1.5.3 +pandas-datareader==0.10.0 +pandas-gbq==0.19.2 +pandas-stubs==2.0.3.230814 +pandocfilters==1.5.1 +panel==1.3.8 +param==2.1.0 +parso==0.8.4 +parsy==2.1 +partd==1.4.2 +pathlib==1.0.1 +pathspec==0.12.1 +patsy==0.5.6 +peewee==3.17.5 +pexpect==4.9.0 +pickleshare==0.7.5 +Pillow==9.4.0 +pip-tools==6.13.0 +platformdirs==4.2.2 +plotly==5.15.0 +plotnine==0.12.4 +pluggy==1.5.0 +polars==0.20.2 +pooch==1.8.1 +portpicker==1.5.2 +prefetch-generator==1.0.3 +preshed==3.0.9 +prettytable==3.10.0 +proglog==0.1.10 +progressbar2==4.2.0 +prometheus_client==0.20.0 +promise==2.3 +prompt-toolkit==3.0.43 +prophet==1.1.5 +proto-plus==1.23.0 +protobuf==3.20.3 +psutil==5.9.5 +psycopg2==2.9.9 +ptyprocess==0.7.0 +py-cpuinfo==9.0.0 +py4j==0.10.9.7 +pyarrow==14.0.2 +pyarrow-hotfix==0.6 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pybtex==0.24.0 +pycocotools==2.0.7 +pycodestyle==2.8.0 +pycparser==2.22 +pydantic==2.7.1 +pydantic_core==2.18.2 +pydata-google-auth==1.8.2 +pydot==1.4.2 +pydot-ng==2.0.0 +pydotplus==2.0.2 +PyDrive==1.3.1 +PyDrive2==1.6.3 +pyerfa==2.0.1.4 +pyflakes==2.4.0 +pygame==2.5.2 +Pygments==2.16.1 +PyGObject==3.42.1 +PyJWT==2.3.0 +pymatgen==2024.1.27 +pymc==5.10.4 +pymongo==4.7.2 +pymystem3==0.2.0 +pynvjitlink-cu12==0.2.3 +PyOpenGL==3.1.7 +pyOpenSSL==24.1.0 +pyparsing==3.1.2 +pyperclip==1.8.2 +pyproj==3.6.1 +pyproject_hooks==1.1.0 +pyshp==2.3.1 +PySocks==1.7.1 +pystacknet==0.0.1 +pytensor==2.18.6 +pytest==7.4.4 +python-apt @ file:///backend-container/containers/python_apt-0.0.0-cp310-cp310-linux_x86_64.whl#sha256=b209c7165d6061963abe611492f8c91c3bcef4b7a6600f966bab58900c63fefa +python-box==7.1.1 +python-dateutil==2.8.2 +python-louvain==0.16 +python-slugify==8.0.4 +python-utils==3.8.2 +pytz==2023.4 +pyviz_comms==3.0.2 +PyWavelets==1.6.0 +PyYAML==6.0.1 +pyzmq==24.0.1 +qdldl==0.1.7.post2 +qudida==0.0.4 +ratelim==0.1.6 +referencing==0.35.1 +regex==2023.12.25 +requests==2.31.0 +requests-oauthlib==1.3.1 +requirements-parser==0.9.0 +rich==13.7.1 +rmm-cu12==24.4.0 +rpds-py==0.18.1 +rpy2==3.4.2 +rsa==4.9 +ruamel.yaml==0.18.6 +ruamel.yaml.clib==0.2.8 +safetensors==0.4.3 +scikit-image==0.19.3 +scikit-learn==1.5.0 +scipy==1.11.4 +scooby==0.10.0 +scs==3.2.4.post1 +seaborn==0.13.1 +SecretStorage==3.3.1 +Send2Trash==1.8.3 +sentencepiece==0.1.99 +shapely==2.0.4 +six==1.16.0 +sklearn-pandas==2.2.0 +smart-open==6.4.0 +sniffio==1.3.1 +snowballstemmer==2.2.0 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy==3.7.4 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spglib==2.4.0 +Sphinx==5.0.2 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +SQLAlchemy==2.0.30 +sqlglot==20.11.0 +sqlparse==0.5.0 +srsly==2.4.8 +stanio==0.5.0 +statsmodels==0.14.2 +StrEnum==0.4.15 +sympy==1.12 +tables==3.8.0 +tabulate==0.9.0 +tbb==2021.12.0 +tblib==3.0.0 +tenacity==8.3.0 +tensorboard==2.15.2 +tensorboard-data-server==0.7.2 +tensorflow @ https://storage.googleapis.com/colab-tf-builds-public-09h6ksrfwbb9g9xv/tensorflow-2.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a2ec79931350b378c1ef300ca836b52a55751acb71a433582508a07f0de57c42 +tensorflow-datasets==4.9.4 +tensorflow-estimator==2.15.0 +tensorflow-gcs-config==2.15.0 +tensorflow-hub==0.16.1 +tensorflow-io-gcs-filesystem==0.37.0 +tensorflow-metadata==1.15.0 +tensorflow-probability==0.23.0 +tensorstore==0.1.45 +termcolor==2.4.0 +terminado==0.18.1 +text-unidecode==1.3 +textblob==0.17.1 +tf-slim==1.1.0 +tf_keras==2.15.1 +thinc==8.2.3 +threadpoolctl==3.5.0 +tifffile==2024.5.10 +tinycss2==1.3.0 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch @ https://download.pytorch.org/whl/cu121/torch-2.3.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=0a12aa9aa6bc442dff8823ac8b48d991fd0771562eaa38593f9c8196d65f7007 +torchaudio @ https://download.pytorch.org/whl/cu121/torchaudio-2.3.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=38b49393f8c322dcaa29d19e5acbf5a0b1978cf1b719445ab670f1fb486e3aa6 +torchsummary==1.5.1 +torchtext==0.18.0 +torchvision @ https://download.pytorch.org/whl/cu121/torchvision-0.18.0%2Bcu121-cp310-cp310-linux_x86_64.whl#sha256=13e1b48dc5ce41ccb8100ab3dd26fdf31d8f1e904ecf2865ac524493013d0df5 +tornado==6.3.3 +tqdm==4.66.4 +traitlets==5.7.1 +traittypes==0.2.1 +transformers==4.41.0 +triton==2.3.0 +tweepy==4.14.0 +typer==0.9.4 +types-pytz==2024.1.0.20240417 +types-setuptools==70.0.0.20240523 +typing_extensions==4.11.0 +tzdata==2024.1 +tzlocal==5.2 +uc-micro-py==1.0.3 +uncertainties==3.1.7 +uritemplate==4.1.1 +urllib3==2.0.7 +vega-datasets==0.9.0 +wadllib==1.3.6 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.0.3 +widgetsnbextension==3.6.6 +wordcloud==1.9.3 +wrapt==1.14.1 +xarray==2023.7.0 +xarray-einstats==0.7.0 +xgboost==2.0.3 +xlrd==2.0.1 +xyzservices==2024.4.0 +yarl==1.9.4 +yellowbrick==1.5 +yfinance==0.2.40 +zict==3.0.0 +zipp==3.18.2 diff --git a/benchmarks/matbench_v0.1_StackNet-mat_v0.2/results.json.gz b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/results.json.gz new file mode 100644 index 00000000..a32dcead Binary files /dev/null and b/benchmarks/matbench_v0.1_StackNet-mat_v0.2/results.json.gz differ