From 649899e36cfad53fb7c6fc2477359f736a3c6adc Mon Sep 17 00:00:00 2001 From: Adomas Baliuka Date: Tue, 23 May 2023 15:40:13 +0200 Subject: [PATCH] Improves metadata and error handling --- .github/workflows/CompatHelper.yml | 45 ++++++++++++++++++++++++++++ Project.toml | 2 +- src/alist.jl | 47 +++++++++++++++++++++--------- src/cscjson.jl | 29 ++++++++++++++---- src/cscmat.jl | 14 +++++++-- test/autogen_ldpc.hpp | 25 ++++++++++++++++ test/ldpc.alist | 22 ++++++++++++++ test/ldpc.bincsc.json | 1 + test/test_alist.jl | 3 ++ test/test_cscjson.jl | 2 +- 10 files changed, 166 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/CompatHelper.yml create mode 100644 test/autogen_ldpc.hpp create mode 100644 test/ldpc.alist create mode 100644 test/ldpc.bincsc.json diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..0918161 --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,45 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +permissions: + contents: write + pull-requests: write +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Check if Julia is already available in the PATH + id: julia_in_path + run: which julia + continue-on-error: true + - name: Install Julia, but only if it is not already available in the PATH + uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: ${{ runner.arch }} + if: steps.julia_in_path.outcome != 'success' + - name: "Add the General registry via Git" + run: | + import Pkg + ENV["JULIA_PKG_SERVER"] = "" + Pkg.Registry.add("General") + shell: julia --color=yes {0} + - name: "Install CompatHelper" + run: | + import Pkg + name = "CompatHelper" + uuid = "aa819f21-2bde-4658-8897-bab36330d9b7" + version = "3" + Pkg.add(; name, uuid, version) + shell: julia --color=yes {0} + - name: "Run CompatHelper" + run: | + import CompatHelper + CompatHelper.main() + shell: julia --color=yes {0} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }} diff --git a/Project.toml b/Project.toml index 37c05d6..b495d00 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LDPCStorage" uuid = "d46d874d-5773-4ce9-8adb-568101dc8882" authors = ["Adomas Baliuka "] -version = "0.3.3" +version = "0.3.4" [deps] DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" diff --git a/src/alist.jl b/src/alist.jl index 1d7a753..0ddc8f8 100644 --- a/src/alist.jl +++ b/src/alist.jl @@ -1,6 +1,13 @@ using SparseArrays using LinearAlgebra +struct InconsistentAlistFileError <: Exception + msg::Any +end + +function Base.showerror(io::IO, e::InconsistentAlistFileError) + print(io, "InconsistentAlistFileError: ", e.msg) +end """ $(SIGNATURES) @@ -12,24 +19,32 @@ function load_alist(file_path::AbstractString; check_redundant=false,) @warn "load_alist called on file with extension '$(file_extension(file_path))', expected '.alist'" end - file = open(file_path, "r") - nVN, nCN = space_sep_ints(readline(file)) - dmax_VN, dmax_CN = space_sep_ints(readline(file)) - var_node_degs = space_sep_ints(readline(file)) - check_node_degs = space_sep_ints(readline(file)) - remaining_lines = readlines(file) - close(file) + local nVN, nCN + local dmax_VN, dmax_CN + local var_node_degs, check_node_degs + local remaining_lines + try + open(file_path, "r") do file + nVN, nCN = space_sep_ints(readline(file)) + dmax_VN, dmax_CN = space_sep_ints(readline(file)) + var_node_degs = space_sep_ints(readline(file)) + check_node_degs = space_sep_ints(readline(file)) + remaining_lines = readlines(file) + end + catch e + throw(InconsistentAlistFileError("Failed to parse '$(abspath(file_path))' as alist file. Reason:\n$e")) + end if length(remaining_lines) != nVN + nCN - error("Number of lines in $file_path is inconcistent with stated matrix size.") + throw(InconsistentAlistFileError("Number of lines in $file_path is inconcistent with stated matrix size.")) end if dmax_CN != maximum(check_node_degs) - error("Alist file $file_path claims: max. CN degree=$dmax_CN but contents give $(maximum(check_node_degs)).") + throw(InconsistentAlistFileError("Alist file $file_path claims: max. CN degree=$dmax_CN but contents give $(maximum(check_node_degs)).")) end if dmax_VN != maximum(var_node_degs) - error("Alist file $file_path claims: max. VN degree=$dmax_CN but contents give $(maximum(var_node_degs)).") + throw(InconsistentAlistFileError("Alist file $file_path claims: max. VN degree=$dmax_CN but contents give $(maximum(var_node_degs)).")) end # parity check matrix @@ -40,7 +55,7 @@ function load_alist(file_path::AbstractString; check_redundant=false,) rows = space_sep_ints(remaining_lines[col_ind]) if check_redundant && length(rows) != var_node_degs[col_ind] - error("Variable node degree in $file_path inconcistent with below data for VN $col_ind.") + throw(InconsistentAlistFileError("Variable node degree in $file_path inconcistent with below data for VN $col_ind.")) end for row_ind in rows @@ -56,19 +71,19 @@ function load_alist(file_path::AbstractString; check_redundant=false,) check_node_degree = length(cols) if check_node_degree != check_node_degs[row_ind] - error("Check node degree in $file_path inconcistent with below data for CN $row_ind.") + throw(InconsistentAlistFileError("Check node degree in $file_path inconcistent with below data for CN $row_ind.")) end entry_counter += check_node_degree for col_ind in cols if H[row_ind, col_ind] != 1 - error("VN and CN specifications in $file_path disagree on matrix entry ($row_ind, $col_ind).") + throw(InconsistentAlistFileError("VN and CN specifications in $file_path disagree on matrix entry ($row_ind, $col_ind).")) end end end if entry_counter != sum(H) - error("VN and CN specification in $file_path are inconsistent.") + throw(InconsistentAlistFileError("VN and CN specification in $file_path are inconsistent.")) end end @@ -80,7 +95,9 @@ end $(SIGNATURES) Save LDPC matrix to file in alist format. For details about the format, see: + https://aff3ct.readthedocs.io/en/latest/user/simulation/parameters/codec/ldpc/decoder.html#dec-h-path-image-required-argument + http://www.inference.org.uk/mackay/codes/alist.html """ function save_to_alist(out_file_path::String, matrix::AbstractArray{Int8,2}) @@ -95,7 +112,9 @@ end $(SIGNATURES) Save LDPC matrix to file in alist format. For details about the format, see: + https://aff3ct.readthedocs.io/en/latest/user/simulation/parameters/codec/ldpc/decoder.html#dec-h-path-image-required-argument + http://www.inference.org.uk/mackay/codes/alist.html """ function print_alist(io::IO, matrix::AbstractArray{Int8,2}) diff --git a/src/cscjson.jl b/src/cscjson.jl index 0b82d10..14b6e28 100644 --- a/src/cscjson.jl +++ b/src/cscjson.jl @@ -2,7 +2,7 @@ using SparseArrays using LinearAlgebra using JSON -const CSCJSON_FORMAT_VERSION = v"0.3.2" # track version of our custom compressed sparse storage json file format. +const CSCJSON_FORMAT_VERSION = v"0.3.3" # track version of our custom compressed sparse storage json file format. const format_if_nnz_values_omitted = :BINCSCJSON const format_if_nnz_values_stored = :COMPRESSED_SPARSE_COLUMN @@ -13,11 +13,22 @@ const description = "Compressed sparse column storage of a matrix. The format de get_metadata() = Dict( - # :julia_package_version => "vTODO", # TODO!!!! USE `pkgversion(m::Module)` IN JULIA 1.9 - :julia_package_url => "https://github.com/XQP-Munich/LDPCStorage.jl", + [ + :julia_package_version => string(pkgversion(LDPCStorage)) + :julia_package_url => "https://github.com/XQP-Munich/LDPCStorage.jl" + ] ) +struct InconsistentBINCSCError <: Exception + msg::Any +end + +function Base.showerror(io::IO, e::InconsistentBINCSCError) + print(io, "InconsistentBINCSCError: ", e.msg) +end + + """ $(SIGNATURES) @@ -57,8 +68,8 @@ function print_bincscjson( ; comments::AbstractString="", ) - all(x->x==1, mat.nzval) || error( - "The input matrix has nonzero entries besides 1. Note: the matrix should have no stored zeros.") + all(x->x==1, mat.nzval) || throw(ArgumentError( + "The input matrix has nonzero entries besides 1. Note: the matrix should have no stored zeros.")) data = Dict( :CSCJSON_FORMAT_VERSION => string(CSCJSON_FORMAT_VERSION), @@ -181,6 +192,12 @@ function load_ldpc_from_json(file_path::AbstractString; expand_qc_exponents_to_b return Hqc end else - error("File $file_path specifies invalid format `$(data["format"])`.") + throw(InconsistentBINCSCError("File $file_path specifies invalid format `$(data["format"])`.")) end end + + +function get_qc_expansion_factor(file_path::AbstractString) + data = JSON.parsefile(file_path) + return Int(data["qc_expansion_factor"]) +end diff --git a/src/cscmat.jl b/src/cscmat.jl index 09d2b36..826d69c 100644 --- a/src/cscmat.jl +++ b/src/cscmat.jl @@ -8,10 +8,15 @@ CSCMAT_FORMAT_VERSION = v"0.1.0" # track version of our custom CSCMAT file form """ Note:THIS FORMATT IS DEPRECATED! USE THE JSON BASED FORMATS! +$(SIGNATURES) + write the three arrays defining compressed sparse column (CSC) storage of a matrix into a file. If `try_hex`, integers in arrays are stored as hexadecimals (without 0x prefix!) If `allow_omit_entries_if_only_stored_ones`, the `stored values` array is omitted if all stored values compare equal to 1. + +additional_header_lines should contain the quasi-cyclic exponent, if any. E.g.: +``additional_header_lines = "QC matrix with expansion factor 32"`` """ function save_to_cscmat( mat::SparseMatrixCSC, destination_file_path::String @@ -77,6 +82,8 @@ end """ Note:THIS FORMATT IS DEPRECATED! USE THE JSON BASED FORMATS! +$(SIGNATURES) + read the three arrays defining compressed sparse column (CSC) storage of a matrix into a file. If `try_hex`, integers are stored as hexadecimals (without 0x prefix!) @@ -142,7 +149,7 @@ end """ -DEPRECATED! +$(SIGNATURES) Convert matrix of exponents for QC LDPC matrix to the actual binary LDPC matrix. @@ -184,6 +191,8 @@ end """ Note:THIS FORMATT IS DEPRECATED! USE THE JSON BASED FORMATS! +$(SIGNATURES) + Load exponents for a QC-LDPC matrix from a `.CSCMAT` file and return the binary LDPC matrix. Not every input `.cscmat` file will give a meaninful result. @@ -192,6 +201,7 @@ Meanwhile, this function expects that the file stores exponents for a quasi-cycl The exponent matrix is read and expanded using the expansion factor. If the expansion factor is not provided, the CSCMAT file must contain a line specifying it. +For example, 'QC matrix with expansion factor 32' """ function load_matrix_from_qc_cscmat_file(file_path::AbstractString; expansion_factor=nothing) if isnothing(expansion_factor) @@ -209,7 +219,7 @@ function load_matrix_from_qc_cscmat_file(file_path::AbstractString; expansion_fa m = match(r"Quasi cyclic exponents for a binary LDPC matrix with expansion factor ([0-9]*)\.", header) if isnothing(m) - error("Failed to infer expansion factor! No header line found containing it.") + throw(InconsistentBINCSCError("Failed to infer expansion factor! No header line found containing it.")) else expansion_factor = parse(Int, m.captures[1]) @info "Inferred expansion factor from file header: $expansion_factor" diff --git a/test/autogen_ldpc.hpp b/test/autogen_ldpc.hpp new file mode 100644 index 0000000..956aa11 --- /dev/null +++ b/test/autogen_ldpc.hpp @@ -0,0 +1,25 @@ +// This file was automatically generated using LDPCStorage.jl (https://github.com/XQP-Munich/LDPCStorage.jl). +// A sparse LDPC matrix (containing only zeros and ones) is saved in compressed sparse column (CSC) format. +// Since the matrix (and LDPC code) is known at compile time, there is no need to save it separately in a file. +// This significantly blows up the executable size (the memory would still have to be used when saving the matrix). + +#include +#include + +namespace AutogenLDPC { + +constexpr inline std::size_t M = 4; +constexpr inline std::size_t N = 14; +constexpr inline std::size_t num_nz = 23; +constexpr inline std::array colptr = { +0x0,0x2,0x3,0x4,0x8,0x9,0xa,0xb,0xc,0xe,0xf,0x10,0x13,0x15,0x17 +}; + +// ------------------------------------------------------- + +constexpr inline std::array row_idx = { +0x1,0x3,0x2,0x0,0x0,0x1,0x2,0x3,0x1,0x2,0x2,0x3,0x0,0x2,0x3,0x1,0x0,0x2,0x3,0x0,0x2,0x1,0x3 +}; + + +} // namespace AutogenLDPC diff --git a/test/ldpc.alist b/test/ldpc.alist new file mode 100644 index 0000000..5a0a206 --- /dev/null +++ b/test/ldpc.alist @@ -0,0 +1,22 @@ +14 4 +4 7 +2 1 1 4 1 1 1 1 2 1 1 3 2 2 +5 5 7 6 +2 4 +3 +1 +1 2 3 4 +2 +3 +3 +4 +1 3 +4 +2 +1 3 4 +1 3 +2 4 +3 4 9 12 13 +1 4 5 11 14 +2 4 6 7 9 12 13 +1 4 8 10 12 14 diff --git a/test/ldpc.bincsc.json b/test/ldpc.bincsc.json new file mode 100644 index 0000000..a6c5058 --- /dev/null +++ b/test/ldpc.bincsc.json @@ -0,0 +1 @@ +{"metadata":{"julia_package_version":"0.3.4","julia_package_url":"https://github.com/XQP-Munich/LDPCStorage.jl"},"comments":"","n_rows":4,"n_columns":14,"rowval":[1,3,2,0,0,1,2,3,1,2,2,3,0,2,3,1,0,2,3,0,2,1,3],"description":"Compressed sparse column storage of a matrix. The format defines a sparse matrix using arrays 'column pointers' (json key `colptr`), 'row indices' (key `rowval`) and 'stored entries of the matrix' (key `nzval`). If the `format` is BINCSCJSON, the `nzval` array is omitted and all non-zero entries of the matrix are assumed to be '1'.If `format` is COMPRESSED_SPARSE_COLUMN, `nzval` is included.\n\nThis file stores a sparse binary matrix in compressed sparse column (CSC) format.","CSCJSON_FORMAT_VERSION":"0.3.3","colptr":[0,2,3,4,8,9,10,11,12,14,15,16,19,21,23],"format":"BINCSCJSON","n_stored_entries":23} \ No newline at end of file diff --git a/test/test_alist.jl b/test/test_alist.jl index 5639d4d..b8afabd 100644 --- a/test/test_alist.jl +++ b/test/test_alist.jl @@ -18,4 +18,7 @@ using LDPCStorage @test H_checked_redundancy == H_loaded @test H == H_loaded + + # check failures + @test_throws Exception load_alist("$(pkgdir(LDPCStorage))/test/files/test_Hqc.cscmat") # completely invalid file end diff --git a/test/test_cscjson.jl b/test/test_cscjson.jl index 4d0172d..395c4a1 100644 --- a/test/test_cscjson.jl +++ b/test/test_cscjson.jl @@ -79,5 +79,5 @@ end target_file = tempname() * ".bincsc.json" - @test_throws ErrorException save_to_bincscjson(target_file, H; comments="Some comment") + @test_throws ArgumentError save_to_bincscjson(target_file, H; comments="Some comment") end