diff --git a/.gitignore b/.gitignore index bf24f99..deb300d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,166 @@ -tests/__pycache__/* -tests/~$* -src/xlsxdatagrid/__pycache__/* +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + src/xlsxdatagrid/__about__.py -.pytest_cache/* reference/example* *.xlsx -reference/*.xlsx -docs/.ipynb_checkpoints/* -.virtual_documents/* \ No newline at end of file diff --git a/src/xlsxdatagrid/colours.py b/src/xlsxdatagrid/colours.py index 3d963b4..c8fe188 100644 --- a/src/xlsxdatagrid/colours.py +++ b/src/xlsxdatagrid/colours.py @@ -9,6 +9,7 @@ GreenOrange_12, ) from pydantic_extra_types.color import Color +import logging BANG_WONG_COLORS = dict( black=Color("RGB(0, 0, 0)"), @@ -52,16 +53,33 @@ def hex_colors(self): def get_color_pallette( length, palettes_in_use, palettes=XLSXDATAGRID_STANDARD_PALLETTES ): + _max = list(XLSXDATAGRID_STANDARD_PALLETTES.keys())[-1][1][1] + _max_pallete = list(palettes.values())[-1][1] for k, v in palettes.items(): if k[0] not in palettes_in_use: if length < k[1][0]: palettes_in_use += [k[0]] return v[0].hex_colors[0:length] - elif k[1][0] > length > k[1][1]: + elif k[1][0] < length <= k[1][1]: palettes_in_use += [k[0]] return v[1].hex_colors[0:length] - else: + elif k[1][1] < length <= _max: pass + elif length > _max: + + logging.warning(f"don't have a colour pallette of length: {length}") + extra = length - _max + if extra > _max: + raise ValueError( + f"error selecting colour pallette of length = {length}" + ) + else: + return _max_pallete.hex_colors + _max_pallete.hex_colors[0:extra] + + else: + raise ValueError( + f"error selecting colour pallette of length = {length}" + ) def color_variant(hex_color, brightness_offset=1): diff --git a/src/xlsxdatagrid/xlsxdatagrid.py b/src/xlsxdatagrid/xlsxdatagrid.py index 7eb189c..f3f9e01 100644 --- a/src/xlsxdatagrid/xlsxdatagrid.py +++ b/src/xlsxdatagrid/xlsxdatagrid.py @@ -29,7 +29,7 @@ However, since it should correspond to the name of the field in the data file it may be important to preserve case.""" -# NOT IN USE +# NOT IN USE ------------------------------- class HeaderStyling(BaseModel): # matches ipydatagrid header_background_color: ty.Optional[Color] = Field( None, description="background color for all non-body cells (index and columns)" @@ -53,30 +53,6 @@ class HeaderStyling(BaseModel): # matches ipydatagrid ) -class Constraints(BaseModel): - minimum: ty.Optional[ty.Union[int, float]] = None - maximum: ty.Optional[ty.Union[int, float]] = None - exclusiveMinimum: ty.Optional[bool] = None - exclusiveMaximum: ty.Optional[bool] = None - enum: ty.Optional[list[ty.Any]] = None - maxLength: ty.Optional[int] = None - minLength: ty.Optional[int] = None - - -NUMERIC_CONSTRAINTS = [ - l for l in list(Constraints.__annotations__.keys()) if l != "enum" -] -LI_CONSTRAINTS = list(Constraints.__annotations__.keys()) -# https://xlsxwriter.readthedocs.io/working_with_data_validation.html#criteria - -MAP_TYPES_JSON_XL = {"integer": "integer", "float": "decimal", "date": "date"} - -PY2XL = { - "**": "^", - "!=": "<>", - # other simple arithemetic operators the same -} - XL_FORMAT_PROPERTIES = ( "font_name", "font_size", @@ -129,6 +105,28 @@ class Constraints(BaseModel): ) # ^these are set at schema level for the whole table +# ^ NOT IN USE ------------------------------- + + +class Constraints(BaseModel): + minimum: ty.Optional[ty.Union[int, float]] = None + maximum: ty.Optional[ty.Union[int, float]] = None + exclusiveMinimum: ty.Optional[bool] = None + exclusiveMaximum: ty.Optional[bool] = None + enum: ty.Optional[list[ty.Any]] = None + maxLength: ty.Optional[int] = None + minLength: ty.Optional[int] = None + + +NUMERIC_CONSTRAINTS = [ + l for l in list(Constraints.__annotations__.keys()) if l != "enum" +] +LI_CONSTRAINTS = list(Constraints.__annotations__.keys()) +# https://xlsxwriter.readthedocs.io/working_with_data_validation.html#criteria + +MAP_TYPES_JSON_XL = {"integer": "integer", "float": "decimal", "date": "date"} + + XL_TABLE_COLUMNS_PROPERTIES = ( "header", "header_format", @@ -150,6 +148,27 @@ class Constraints(BaseModel): TIME_FORMAT = {"num_format": TIME_STR} DURATION_FORMAT = {"num_format": DURATION_STR} +PY2XL = { + "**": "^", + "!=": "<>", + # other simple arithemetic operators the same +} + + +def py2xl_formula(formula, map_names): + def replace(formula, di): + + for k, v in di.items(): + if k in formula: + formula = formula.replace(k, v) + return formula + + map_table_names = {l: f"[@[{l}]]" for l in map_names.keys()} + formula = replace(formula, PY2XL) + formula = replace(formula, map_names) + formula = replace(formula, map_table_names) + return "= " + formula + def get_numeric_constraints(di): return [k for k in di.keys() if k in NUMERIC_CONSTRAINTS] @@ -188,7 +207,7 @@ class FieldSchemaXl(FieldSchema): xl_formula: ty.Optional[str] = None -def get_xl_constraints(f: FieldSchema): +def get_xl_constraints(f: FieldSchema): # TODO: write text for this if f.type == "boolean": return { "validate": "list", @@ -340,21 +359,6 @@ def field_names(self) -> list: return [f.name for f in self.fields] -def py2xl_formula(formula, map_names): - def replace(formula, di): - - for k, v in di.items(): - if k in formula: - formula = formula.replace(k, v) - return formula - - map_table_names = {l: f"[@[{l}]]" for l in map_names.keys()} - formula = replace(formula, PY2XL) - formula = replace(formula, map_names) - formula = replace(formula, map_table_names) - return "= " + formula - - def convert_date_to_excel_ordinal(d: date, offset: int = 693594): # the offset date value for the date of 1900-01-00 = 693594 return d.toordinal() - offset @@ -412,13 +416,6 @@ class XlTableWriter(BaseModel): @model_validator(mode="after") def build(self) -> "XlTableWriter": - # ensure data and key col names in same order - if self.gridschema.field_names != list(self.data.keys()): - self.data = { - l: self.data[l] - for l in self.gridschema.field_names - if l in self.data.keys() - } self.metadata = self.gridschema.metadata_fstring.format( **self.gridschema.model_dump() @@ -427,10 +424,21 @@ def build(self) -> "XlTableWriter": is_t = self.gridschema.is_transposed ix_nm = self.gridschema.datagrid_index_name # column headings hd = self.gridschema.header_depth # header depth - fd_nns = list(self.data.keys()) # field names + fd_nns = self.gridschema.field_names # field names length = len(self.data[fd_nns[0]]) - 1 # length of data arrays self.format_headers = hd * [None] + # ensure data and key col names in same order + if self.gridschema.field_names != list(self.data.keys()): + self.data = { + l: (lambda l, data: data[l] if l in data.keys() else [None] * length)( + l, self.data + ) + for l in self.gridschema.field_names + } + assert self.gridschema.field_names == list(self.data.keys()) + # TODO: allow option of only outputting fields which have data associated with them + if is_t: x += 1 else: @@ -675,9 +683,6 @@ def write_table(workbook, xl_tbl: XlTableWriter): for k, v in xl_tbl.format_arrays.items(): columns[k]["format"] = formats[v] # ^ TODO: formatting dates and datetime as numeric with excel string formatting - columns["a_int"]["format"] = workbook.add_format( - {"num_format": "[$$-409]#,##0.00"} - ) options = dict( style="Table Style Light 1", header_row=True, @@ -685,9 +690,10 @@ def write_table(workbook, xl_tbl: XlTableWriter): columns=list(columns.values()), ) - options = options | {"name": name} + options = options # | {"name": name} # TODO: <- table name needs to not inc. spaces etc... update + # ^ a known table name will be important if / when we want to do lookups between tables... + worksheet.add_table(*xl_tbl.tbl_range, options) - # worksheet.add_table(*xl_tbl.tbl_range, options) # NOTE: if you write a table to excel with a header - the table range includes the header. # ------------------------------------- @@ -748,7 +754,7 @@ def write_table(workbook, xl_tbl: XlTableWriter): worksheet.hide_gridlines(xl_tbl.hide_gridlines) # write metadata worksheet.write(*xl_tbl.xy, xl_tbl.metadata, header_label_cell_format) - + # worksheet.add_table(*xl_tbl.tbl_range, options) return None @@ -764,6 +770,13 @@ def from_jsonschema_and_data(data: dict, gridschema: dict, fpth: pathlib.Path = return fpth +def from_jsonschemas_and_datas( + data: list[dict], gridschema: list[dict], fpth: pathlib.Path = None +): + assert len(data) == len(gridschema) + pass + + def from_pydantic_object( pydantic_object: ty.Type[BaseModel], fpth: pathlib.Path = None ) -> pathlib.Path: diff --git a/tests/constants.py b/tests/constants.py index dee4389..5310959 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -1,10 +1,9 @@ import pathlib - -PATH_XL = pathlib.Path(__file__).parent / "ExcelOut.xlsx" -PATH_XL_MANY_SHEETS = pathlib.Path(__file__).parent / "ExcelOutManySheets.xlsx" -PATH_XL_TRANSPOSED = pathlib.Path(__file__).parent / "ExcelOutTransposed.xlsx" -PATH_XL_FROM_SCHEMA = pathlib.Path(__file__).parent / "ExcelOutFromSchema.xlsx" -PATH_XL_FROM_SCHEMA_TRANSPOSED = ( - pathlib.Path(__file__).parent / "ExcelOutFromSchemaTransposed.xlsx" -) +FDIR = pathlib.Path(__file__).parent +PATH_XL = FDIR / "ExcelOut.xlsx" +PATH_XL_MANY_SHEETS = FDIR / "ExcelOutManySheets.xlsx" +PATH_XL_TRANSPOSED = FDIR / "ExcelOutTransposed.xlsx" +PATH_XL_FROM_SCHEMA = FDIR / "ExcelOutFromSchema.xlsx" +PATH_XL_FROM_SCHEMA_TRANSPOSED = FDIR / "ExcelOutFromSchemaTransposed.xlsx" +PATH_XL_FROM_API = FDIR / "ExcelOutFromApi.xlsx" diff --git a/tests/test_xlsxdatagrid.py b/tests/test_xlsxdatagrid.py index c046852..60488f1 100644 --- a/tests/test_xlsxdatagrid.py +++ b/tests/test_xlsxdatagrid.py @@ -1,3 +1,5 @@ +import requests +import pathlib from enum import Enum from typing_extensions import Annotated from datetime import date, datetime, time, timedelta @@ -9,12 +11,13 @@ ConfigDict, computed_field, StringConstraints, - PlainSerializer, NaiveDatetime, # NaiveDate, ) import pytest import xlsxwriter as xw +import jsonref + from .constants import ( PATH_XL, @@ -22,6 +25,7 @@ PATH_XL_TRANSPOSED, PATH_XL_FROM_SCHEMA, PATH_XL_FROM_SCHEMA_TRANSPOSED, + PATH_XL_FROM_API, ) from xlsxdatagrid.xlsxdatagrid import ( write_table, @@ -29,7 +33,7 @@ convert_records_to_datagrid_schema, XlTableWriter, DataGridSchema, - convert_date_to_excel_ordinal, + convert_list_records_to_dict_arrays, ) @@ -324,3 +328,33 @@ def test_schema_and_data_write_table(is_transposed): write_table(workbook, xl_tbl) workbook.close() assert fpth_xl.is_file() + + +def test_schema_and_data_from_digital_schedules_api(): + fpth_xl = PATH_XL_FROM_API + response = requests.get( + "https://aectemplater-dev.maxfordham.com/type_specs/project_revision/1/object/602/grid?override_units=true" + ) + assert ( + response.status_code == 200 + ), f"API request failed with status code {response.status_code}" + + fpth_xl = pathlib.Path("./test.xlsx") + data = jsonref.replace_refs(response.json()) + data["data"] = data["data"] + data["data"] + data_array = convert_list_records_to_dict_arrays(data["data"]) + + gridschema = convert_records_to_datagrid_schema(data["$schema"]) + # HOTFIX: Replace all anyOfs with the first type + for field in gridschema["fields"]: + if "anyOf" in field.keys(): + field["type"] = field["anyOf"][0]["type"] + field.pop("anyOf") + gridschema["datagrid_index_name"] = ("section", "unit", "name") + gridschema["is_transposed"] = True + + xl_tbl = XlTableWriter(gridschema=gridschema, data=data_array) + workbook = xw.Workbook(str(fpth_xl)) + write_table(workbook, xl_tbl) + workbook.close() + assert fpth_xl.is_file()