diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..33afdff --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +target +docs +examples +pg_dump_anon +**/_build \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39533dd..2df4d79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,3 +47,9 @@ include: - local: .gitlab/job_templates/coverage.yml inputs: pgver: pg17 + ## Dalibo Specific jobs + - local: dalibo/.gitlab-ci.yml + rules: + - exists: + - dalibo/.gitlab-ci.yml + diff --git a/.gitlab/issue_templates/Release.md b/.gitlab/issue_templates/Release.md new file mode 100644 index 0000000..16129aa --- /dev/null +++ b/.gitlab/issue_templates/Release.md @@ -0,0 +1,28 @@ + +* [ ] Check that **all** CI jobs run without errors on the `latest` branch +* [ ] Close all remaining issues on the current milestone +* [ ] Update the [Changelog] +* [ ] Update the [AUTHORS.md] list +* [ ] Write the announcement in [NEWS.md] +* [ ] Rebuild the docker image `latest` and upload it + (`make docker_image docker_push`) +* [ ] Close the current milestone and open the next one +* [ ] Rebase the `stable` branch from `latest` +* [ ] Rebuild the docker image `stable` and upload it + (`DOCKER_TAG=stable make docker_image docker_push`) +* [ ] Tag the `latest` branch +* [ ] Build the docker image `x.y.z` and upload it + (`DOCKER_TAG=x.y.z make docker_image docker_push`) +* [ ] Create a branch `x.y.z` based on `stable` +* [ ] Launch [a new pipeline] on the `x.y.z` branch +* [ ] Once the pipeline is done, trigger the deploy task to publish the packages +* [ ] Update the upstream repositories +* [ ] On the [Tags page], click on `Create a Release` +* [ ] Bump to the new version number in [Cargo.toml] +* [ ] Publish the announcement + +[Changelog]: CHANGELOG.md +[NEWS.md]: NEWS.md +[Cargo.toml]: Cargo.toml +[Tags page]: https://gitlab.com/dalibo/postgresql_anonymizer/-/tags/ +[a new pipeline]: https://gitlab.com/dalibo/postgresql_anonymizer/-/pipelines/new diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81f80ab..460f125 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: rev: v0.13.0 hooks: - id: markdownlint - exclude: docs/(dev|how-to) + exclude: docs/(dev|how-to|tutorials|runbooks) - repo: local hooks: - id: rustfmt diff --git a/AUTHORS.md b/AUTHORS.md index c2a99b7..66064e7 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -16,8 +16,12 @@ help of many contributors. Contributors ------------------------------------------------------------------------------- +* Pierre Giraud: Proofreading +* Benoit Lobréau: Prototyping, Code Review +* Robin Portigliatti: Tutorials +* Suhas Thalanki: Build on Mac OS X * Daniel Solsona: Feature idea -* Jukka Heiskanen: Bug report and analysis +* Jukka Heiskanen: Bug reports and analysis * Julien Acroute: Documentation * Philip Olson: Documentation * Anthony Dumontois: Documentation @@ -72,7 +76,6 @@ Contributors * Matthieu Larcher (@somatt) : Bug reports * Nikolay Samokhvalov (@NikolayS) : Bug fix + Documentation * Gunnar "Nick" Bluth (@nickbluth) : Some additional functions -* Yann ROBIN (@me.show) : version() function and DBAAS install * Ilya Gorbunov (@dAverk) : Bug fixes * Peter Neave (@peterneave) : Typos * Yann Robin (@me.show) : version() function and DBAAS install diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e53e8..67fe4ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,16 @@ CHANGELOG -WIP : 2.3.0 - Parallel Static Masking +20250702 : 2.3.0 - Parallel Static Masking ------------------------------------------------------------------------------- +* [docs] Various corrections on the tutorial (Robin Portigliatti) +* [random] FIX #527 issue with negative numbers +* [make] build on Mac OS X (Suhas Thalanki) +* [doc] How to truncate a table for masked users +* [replica] Introducing Replica Masking (ALPHA) +* [tests] make ldm test more robust +* [doc] hash and unescaped chars 20250530 : 2.2.1 - packaging fixup ------------------------------------------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 747fac7..dcbb027 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ Where to start ? If you want to help, here's a few ideas : -1- **Testing** : You can install the `master` branch of the project and realize +1- **Testing** : You can install the `latest` branch of the project and realize extensive tests based on your use case. This is very useful to improve the stability of the code. Eventually if you can publish you test cases, please add them in the `/tests/sql` directory or in `demo`. I have recently @@ -54,17 +54,17 @@ Add a new remote to your local repo: git remote add upstream https://gitlab.com/dalibo/postgresql_anonymizer.git ``` -### Keep your master branch up to date +### Keep your `latest` branch up to date At any time, you can mirror your personal repo like this: ```bash -# switch to the master branch -git checkout master -# download the latest commit from the main repo +# switch to the latest branch +git checkout latest +# download the latest commits from the upstream repo git fetch upstream -# apply the latest commits -git rebase upstream/master +# apply the commits +git rebase upstream/latest # push the changes to your personal repo git push origin ``` @@ -82,7 +82,7 @@ git checkout foo # download the latest commit from the main repo git fetch upstream # apply the latest commits -git rebase upstream/master +git rebase upstream/latest # push the changes to your personal repo git push origin --force-with-lease ``` @@ -385,19 +385,4 @@ they are not `SECURITY DEFINER`. Publishing a new Release -------------------------------------------------------------------------------- -* [ ] Check that **all** CI jobs run without errors on the `master` branch -* [ ] Close all remaining issues on the current milestone -* [ ] Update the [Changelog] -* [ ] Write the announcement in [NEWS.md] -* [ ] Rebuild the docker image and upload it (`make docker_image docker_push`) -* [ ] Upload the zipball to PGXN -* [ ] Close the current milestone and open the next one -* [ ] Tag the `latest` branch -* [ ] Rebase the `stable` branch from `latest` -* [ ] Publish the RPM/DEB packages -* [ ] Bump to the new version number in [Cargo.toml] -* [ ] Publish the announcement - -[Changelog]: CHANGELOG.md -[NEWS.md]: NEWS.md -[Cargo.toml]: Cargo.toml +See .gitlab/issue_templates/Release.md diff --git a/Cargo.lock b/Cargo.lock index eb36ebe..11d7373 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 4 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" @@ -28,9 +28,12 @@ dependencies = [ [[package]] name = "aligned-vec" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] [[package]] name = "allocator-api2" @@ -65,12 +68,14 @@ dependencies = [ [[package]] name = "anon" -version = "2.2.0" +version = "2.2.1" dependencies = [ "c_str_macro", "chrono", "fake", + "fastrand", "image", + "libc", "md-5", "paste", "pgrx", @@ -81,15 +86,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arbitrary" @@ -116,9 +121,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", @@ -127,9 +132,9 @@ dependencies = [ [[package]] name = "atomic" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" dependencies = [ "bytemuck", ] @@ -146,15 +151,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "av1-grain" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +checksum = "4f3efb2ca85bc610acfa917b5aaa36f3fcbebed5b3182d7f877b02531c4b80c8" dependencies = [ "anyhow", "arrayvec", @@ -166,18 +171,18 @@ dependencies = [ [[package]] name = "avif-serialize" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e335041290c43101ca215eed6f43ec437eb5a42125573f600fc3fa42b9bddd62" +checksum = "19135c0c7a60bfee564dbe44ab5ce0557c6bf3884e5291a50be76a15640c4fbd" dependencies = [ "arrayvec", ] [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -185,7 +190,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -196,9 +201,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bigdecimal" -version = "0.4.5" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d712318a27c7150326677b321a5fa91b55f6d9034ffd67f20319e147d40cee" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ "autocfg", "libm", @@ -214,7 +219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ "annotate-snippets", - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -228,18 +233,18 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.6.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bit_field" @@ -255,9 +260,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bitstream-io" @@ -288,21 +293,21 @@ dependencies = [ [[package]] name = "built" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" [[package]] name = "byteorder" @@ -318,9 +323,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.7.2" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "c_str_macro" @@ -330,18 +335,18 @@ checksum = "c6d44951c469019e225e7667d799052f67fb8ea358d086878f3582b39f0de5e5" [[package]] name = "camino" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +checksum = "0da45bc31171d8d6960122e222a67740df867c1dd53b4d51caa297084c185cab" dependencies = [ "serde", ] [[package]] name = "cargo-platform" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" dependencies = [ "serde", ] @@ -357,7 +362,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 1.0.63", + "thiserror 1.0.69", ] [[package]] @@ -372,9 +377,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.19" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "jobserver", "libc", @@ -412,22 +417,22 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets", + "windows-link", ] [[package]] @@ -443,9 +448,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -464,9 +469,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstyle", "clap_lex", @@ -474,9 +479,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck", "proc-macro2", @@ -486,9 +491,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "color_quant" @@ -513,9 +518,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -531,9 +536,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -550,15 +555,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" @@ -572,18 +577,18 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] [[package]] name = "deunicode" -version = "1.6.0" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" [[package]] name = "digest" @@ -609,9 +614,9 @@ dependencies = [ [[package]] name = "either" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "enum-map" @@ -633,20 +638,40 @@ dependencies = [ "syn", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -656,7 +681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" dependencies = [ "bit_field", - "half 2.4.1", + "half 2.6.0", "lebe", "miniz_oxide", "rayon-core", @@ -684,7 +709,7 @@ dependencies = [ "chrono", "deunicode", "http", - "rand 0.9.0", + "rand 0.9.1", "random_color", "rust_decimal", "time", @@ -700,9 +725,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fdeflate" @@ -721,9 +746,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "miniz_oxide", @@ -758,9 +783,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -768,15 +793,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -785,21 +810,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-macro", @@ -822,20 +847,20 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", ] [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", @@ -845,9 +870,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" dependencies = [ "color_quant", "weezl", @@ -855,15 +880,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "half" @@ -873,9 +898,9 @@ checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" [[package]] name = "half" -version = "2.4.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -892,15 +917,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.15.2" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ "allocator-api2", "equivalent", @@ -925,15 +944,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - -[[package]] -name = "hermit-abi" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hmac" @@ -955,9 +968,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -966,16 +979,17 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core 0.61.2", ] [[package]] @@ -989,21 +1003,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -1012,31 +1027,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -1044,67 +1039,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "idna" version = "1.0.3" @@ -1118,9 +1100,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -1128,9 +1110,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.5" +version = "0.25.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" +checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" dependencies = [ "bytemuck", "byteorder-lite", @@ -1151,9 +1133,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e031e8e3d94711a9ccb5d6ea357439ef3dcbed361798bd4071dc4d9793fbe22f" +checksum = "f6970fe7a5300b4b42e62c52efa0187540a5bef546c60edaf554ef595d2e6f0b" dependencies = [ "byteorder-lite", "quick-error 2.0.1", @@ -1173,12 +1155,12 @@ checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" [[package]] name = "indexmap" -version = "2.5.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown", ] [[package]] @@ -1194,13 +1176,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ - "hermit-abi 0.4.0", + "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1229,31 +1211,33 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.3", "libc", ] [[package]] name = "jpeg-decoder" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -1271,15 +1255,15 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libfuzzer-sys" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" dependencies = [ "arbitrary", "cc", @@ -1287,19 +1271,19 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets", + "windows-targets 0.53.2", ] [[package]] name = "libm" -version = "0.2.8" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "linux-raw-sys" @@ -1309,15 +1293,15 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -1325,9 +1309,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "loop9" @@ -1360,9 +1344,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "minimal-lexical" @@ -1372,9 +1356,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "simd-adler32", @@ -1382,14 +1366,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ - "hermit-abi 0.3.9", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -1477,23 +1460,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] name = "objc2-core-foundation" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daeaf60f25471d26948a1c2f840e3f7d86f4109e3af4e8e4b5cd70c39690d925" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] name = "object" -version = "0.36.4" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -1506,19 +1488,19 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "owo-colors" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" dependencies = [ "supports-color 2.1.0", - "supports-color 3.0.1", + "supports-color 3.0.2", ] [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -1526,15 +1508,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1561,12 +1543,12 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a98c6720655620a521dcc722d0ad66cd8afd5d86e34a89ef691c50b7b24de06" +checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", - "hashbrown 0.15.2", + "hashbrown", "indexmap", "serde", ] @@ -1574,11 +1556,10 @@ dependencies = [ [[package]] name = "pgrx" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dffb5a99b514090574a668078a28394317d30bbb33a8b6d651342ebd945e81b" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "atomic-traits", - "bitflags 2.9.0", + "bitflags 2.9.1", "bitvec", "enum-map", "heapless", @@ -1598,8 +1579,7 @@ dependencies = [ [[package]] name = "pgrx-bindgen" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e43f241932f230af1ed713fac06133a074efb12a80705db7f679627551b5a8" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "bindgen", "cc", @@ -1617,8 +1597,7 @@ dependencies = [ [[package]] name = "pgrx-macros" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c97650c7bb75290fc0b7e1a80bc5cea74e541015369a6558f72fb3da0c0e95e" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "pgrx-sql-entity-graph", "proc-macro2", @@ -1629,8 +1608,7 @@ dependencies = [ [[package]] name = "pgrx-pg-config" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a370f7f6127487e50be58aaf6544439352ff667e9ab6e5e6b24fb77f794315b" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "cargo_toml", "eyre", @@ -1647,8 +1625,7 @@ dependencies = [ [[package]] name = "pgrx-pg-sys" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa062f644530005641b5cdc4b5d1303c70ffa38bd1089699035ea46adac41b6d" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "cee-scape", "libc", @@ -1662,8 +1639,7 @@ dependencies = [ [[package]] name = "pgrx-sql-entity-graph" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58ca1db80bc9b38e38be7d65e3dd6ba2813083410e7a81ed9db70147c40f86" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "convert_case", "eyre", @@ -1678,8 +1654,7 @@ dependencies = [ [[package]] name = "pgrx-tests" version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ec4957e116035f87c2fb475b7c95554b44400d66e25842d6edb2491ed6c883" +source = "git+https://github.com/pgcentralfoundation/pgrx.git?branch=develop#c0de15d29d2ef175f3b56b630f85af09448094e5" dependencies = [ "clap-cargo", "eyre", @@ -1691,7 +1666,7 @@ dependencies = [ "pgrx-pg-config", "postgres", "proptest", - "rand 0.9.0", + "rand 0.9.1", "regex", "serde", "serde_json", @@ -1704,27 +1679,27 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -1734,9 +1709,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "png" @@ -1778,7 +1753,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.9.0", + "rand 0.9.1", "sha2", "stringprep", ] @@ -1794,6 +1769,15 @@ dependencies = [ "postgres-protocol", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1802,11 +1786,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.7.35", + "zerocopy", ] [[package]] @@ -1820,18 +1804,18 @@ dependencies = [ [[package]] name = "profiling" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" dependencies = [ "profiling-procmacros", ] [[package]] name = "profiling-procmacros" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" dependencies = [ "quote", "syn", @@ -1839,17 +1823,17 @@ dependencies = [ [[package]] name = "proptest" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.9.0", + "bitflags 2.9.1", "lazy_static", "num-traits", - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand 0.9.1", + "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -1889,9 +1873,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radium" @@ -1912,13 +1896,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.24", ] [[package]] @@ -1947,7 +1930,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -1956,16 +1939,16 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", ] [[package]] name = "rand_xorshift" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.3", ] [[package]] @@ -1974,7 +1957,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d635c5e80ae160390ac62ca027d2d06c94c1dc69e5c0a12f1e3a53664dc84966" dependencies = [ - "rand 0.9.0", + "rand 0.9.1", ] [[package]] @@ -2007,16 +1990,16 @@ dependencies = [ "rand_chacha 0.3.1", "simd_helpers", "system-deps", - "thiserror 1.0.63", + "thiserror 1.0.69", "v_frame", "wasm-bindgen", ] [[package]] name = "ravif" -version = "0.11.11" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2413fd96bd0ea5cdeeb37eaf446a22e6ed7b981d792828721e74ded1980a45c6" +checksum = "5825c26fddd16ab9f515930d49028a630efec172e903483c94796cfe31893e6b" dependencies = [ "avif-serialize", "imgref", @@ -2049,11 +2032,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -2093,9 +2076,9 @@ checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" [[package]] name = "rust_decimal" -version = "1.37.1" +version = "1.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7de2ba56ac291bd90c6b9bece784a52ae1411f9506544b3eae36dd2356d50" +checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" dependencies = [ "arrayvec", "num-traits", @@ -2103,9 +2086,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -2124,17 +2107,23 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys", "windows-sys 0.59.0", ] +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + [[package]] name = "rusty-fork" version = "0.3.0" @@ -2149,9 +2138,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -2176,18 +2165,18 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "semver" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.210" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -2204,9 +2193,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", @@ -2215,9 +2204,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -2227,9 +2216,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -2242,9 +2231,9 @@ checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -2274,30 +2263,27 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.13.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2344,18 +2330,18 @@ dependencies = [ [[package]] name = "supports-color" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8775305acf21c96926c900ad056abeef436701108518cf890020387236ac5a77" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" dependencies = [ "is_ci", ] [[package]] name = "syn" -version = "2.0.100" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -2364,9 +2350,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", @@ -2413,12 +2399,12 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.2", + "getrandom 0.3.3", "once_cell", "rustix", "windows-sys 0.59.0", @@ -2426,11 +2412,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.63", + "thiserror-impl 1.0.69", ] [[package]] @@ -2444,9 +2430,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -2477,9 +2463,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -2492,15 +2478,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -2508,9 +2494,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -2518,9 +2504,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -2533,9 +2519,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -2565,7 +2551,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.1", "socket2", "tokio", "tokio-util", @@ -2574,9 +2560,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -2587,9 +2573,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.19" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -2599,31 +2585,38 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "unarray" @@ -2639,15 +2632,15 @@ checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" @@ -2660,9 +2653,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-segmentation" @@ -2672,9 +2665,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "url" @@ -2696,12 +2689,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -2710,21 +2697,23 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "atomic", - "getrandom 0.3.2", + "getrandom 0.3.3", + "js-sys", "md-5", "sha1_smol", + "wasm-bindgen", ] [[package]] name = "v_frame" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" dependencies = [ "aligned-vec", "num-traits", @@ -2745,9 +2734,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", ] @@ -2764,9 +2753,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -2785,24 +2774,24 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -2811,9 +2800,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2821,9 +2810,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -2834,15 +2823,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -2850,15 +2842,15 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" [[package]] name = "whoami" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ "redox_syscall", "wasite", @@ -2903,28 +2895,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" dependencies = [ "windows-core 0.57.0", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] name = "windows-core" -version = "0.52.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" dependencies = [ - "windows-targets", + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", + "windows-targets 0.52.6", ] [[package]] name = "windows-core" -version = "0.57.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-targets", + "windows-implement 0.60.0", + "windows-interface 0.59.1", + "windows-link", + "windows-result 0.3.4", + "windows-strings", ] [[package]] @@ -2938,6 +2934,17 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-interface" version = "0.57.0" @@ -2949,13 +2956,48 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-result" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", ] [[package]] @@ -2964,7 +3006,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2973,7 +3015,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", ] [[package]] @@ -2982,14 +3033,30 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -2998,53 +3065,101 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" -version = "0.6.18" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ "memchr", ] @@ -3055,20 +3170,14 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "wyz" @@ -3081,9 +3190,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -3093,9 +3202,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", @@ -3105,39 +3214,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ - "zerocopy-derive 0.8.24", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", @@ -3165,11 +3253,22 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -3178,9 +3277,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", @@ -3204,9 +3303,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" +checksum = "7384255a918371b5af158218d131530f694de9ad3815ebdd0453a940485cb0fa" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index 07d6754..0452207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "anon" -version = "2.2.1" +version = "2.3.0" edition = "2021" [lib] @@ -21,15 +21,17 @@ pg_test = [] c_str_macro = "1.0.3" chrono = "0.4.37" fake = { version = "4.3.0", features = ["bigdecimal", "chrono", "http", "rust_decimal", "uuid", "time","random_color"] } +fastrand = "2.3.0" image = "0.25.5" +libc = "0.2.172" md-5 = "0.10.6" paste = "1.0" -pgrx = "0.14.3" +pgrx = { git = "https://github.com/pgcentralfoundation/pgrx.git", branch = "develop" } rand = "0.8.5" regex = "1.10.2" [dev-dependencies] -pgrx-tests = "0.14.3" +pgrx-tests = { git = "https://github.com/pgcentralfoundation/pgrx.git", branch = "develop" } [profile.dev] panic = "unwind" diff --git a/Makefile b/Makefile index 6348a36..c46857a 100644 --- a/Makefile +++ b/Makefile @@ -12,13 +12,30 @@ ANON_MINOR_VERSION?=$(shell grep '^version *= *' Cargo.toml | sed 's/^version *= # use `TARGET=debug make run` for more detailed errors TARGET?=release -TARGET_DIR?=target/$(TARGET)/anon-$(PGVER)/ +ifeq ($(shell uname -s),Darwin) + TARGET_DIR?=target/$(TARGET)/anon-pg$(PG_MAJOR_VERSION) + LIB_SUFFIX?=dylib +else + TARGET_DIR?=target/$(TARGET)/anon-$(PG_MAJOR_VERSION) + LIB_SUFFIX?=so +endif +LIB=anon.$(LIB_SUFFIX) PG_CONFIG?=`$(PGRX) info pg-config $(PGVER) 2> /dev/null || echo pg_config` PG_SHAREDIR?=$(shell $(PG_CONFIG) --sharedir) PG_LIBDIR?=$(shell $(PG_CONFIG) --libdir) PG_PKGLIBDIR?=$(shell $(PG_CONFIG) --pkglibdir) PG_BINDIR?=$(shell $(PG_CONFIG) --bindir) +ifeq ($(shell uname -s),Darwin) + LIB_SUFFIX?=dylib +else + LIB_SUFFIX?=so +endif +LIB=anon.$(LIB_SUFFIX) + +# The instance +PGDATA_DIR=~/.pgrx/data-$(PG_MAJOR_VERSION) + # Be sure to use the PGRX version (PGVER) of the postgres binaries # It's especially important for the pg_dump test in pg_regress PATH:=$(PG_BINDIR):${PATH} @@ -88,6 +105,9 @@ REGRESS_TESTS+= sampling REGRESS_TESTS+= shuffle REGRESS_TESTS+= syntax_checks REGRESS_TESTS+= ternary +# The `test_` is here to avoid collision with the files in the `sql` folder +# DO NOT rename the `tests/sql/test_*` files ! +REGRESS_TESTS+= test_replica_masking REGRESS_TESTS+= test_static_masking REGRESS_TESTS+= transparent_dynamic_masking REGRESS_TESTS+= trusted_schemas @@ -140,7 +160,7 @@ extension: install: cp -r $(TARGET_SHAREDIR)/extension/* $(PG_SHAREDIR)/extension/ - install $(TARGET_PKGLIBDIR)/anon.so $(PG_PKGLIBDIR) + install $(TARGET_PKGLIBDIR)/$(LIB) $(PG_PKGLIBDIR) ## ## INSTALLCHECK @@ -152,8 +172,9 @@ install: # With PGRX: the postgres instance is created previously by `cargo run`. This # means we have some extra tasks to prepare the instance -installcheck: start +installcheck: stop start dropdb $(PSQL_OPT) --if-exists $(PGDATABASE) + dropdb $(PSQL_OPT) --if-exists $(PGDATABASE)_source createdb $(PSQL_OPT) $(PGDATABASE) dropuser oscar_the_owner || echo 'ignored' createuser $(PSQL_OPT) postgres --superuser || echo 'ignored' @@ -180,12 +201,15 @@ test: $(PGRX) test $(PGVER) $(RELEASE_OPT) --verbose start: + sed --in-place 's/^#\?wal_level = .*/wal_level = logical/' $(PGDATA_DIR)/postgresql.conf $(PGRX) start $(PGVER) stop: $(PGRX) stop $(PGVER) run: + # ensure that the wal_level is properly set + sed --in-place 's/^#\?wal_level = .*/wal_level = logical/' $(PGDATA_DIR)/postgresql.conf $(PGRX) run $(PGVER) $(RELEASE_OPT) psql: @@ -264,7 +288,8 @@ package: ## D O C K E R ## -DOCKER_IMAGE?=registry.gitlab.com/dalibo/postgresql_anonymizer +DOCKER_TAG?=latest +DOCKER_IMAGE?=registry.gitlab.com/dalibo/postgresql_anonymizer:$(DOCKER_TAG) ifneq ($(DOCKER_PG_MAJOR_VERSION),) DOCKER_BUILD_ARG := --build-arg DOCKER_PG_MAJOR_VERSION=$(DOCKER_PG_MAJOR_VERSION) @@ -304,3 +329,9 @@ docker_init: #: start a docker container lint: cargo clippy --release + + +## +## DALIBO-specific Makefile +## +-include dalibo/Makefile diff --git a/NEWS.md b/NEWS.md index b3884b2..61d809d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,104 @@ +PostgreSQL Anonymizer 2.3 : Replica Masking (ALPHA) +================================================================================ + +Eymoutiers, France, July 2nd, 2025 + +We're publishing `PostgreSQL Anonymizer 2.3` today, introducing the long awaited +replica masking mechanism. Database Administrators can now synchronize a "masked +clone" with their production database using PostgreSQL logical replication. + +Enhanced Privacy Protection for Your Data +-------------------------------------------------------------------------------- + +`PostgreSQL Anonymizer` is an extension that hides or replaces personally +identifiable information (PII) or commercially sensitive data from a PostgreSQL +database. + +The extension offers five different masking strategies: + +* [Dynamic Masking] - Real-time data protection +* [Static Masking] - Permanent data transformation +* [Replica Masking] - Anonymized logical replication +* [Backup Masking] - Privacy-protected database exports +* [Masking Views] - Controlled data visibility +* [Masking Data Wrappers] - Extended protection across systems + +Each strategy is complemented by an enhanced suite of Masking Functions, including +advanced techniques such as: Substitution, Randomization, Faking, Pseudonymization, +Partial Scrambling, Shuffling, Noise Addition and Generalization. + +The extension can be installed with Debian and RPM packages, an Ansible role, a docker +image, etc. It is also available on major DBaaS providers including : Alibaba Cloud, +Crunchy Bridge, Google Cloud SQL, Microsoft Azure Database, Neon, etc. + +See the [INSTALL] section of the documentation for more details! + + +[Masking Functions]: https://postgresql-anonymizer.readthedocs.io/en/latest/masking_functions/ +[Backup Masking]: https://postgresql-anonymizer.readthedocs.io/en/latest/anonymous_dumps/ +[Static Masking]: https://postgresql-anonymizer.readthedocs.io/en/latest/static_masking/ +[Dynamic Masking]: https://postgresql-anonymizer.readthedocs.io/en/latest/dynamic_masking/ +[Replica Masking]: https://postgresql-anonymizer.readthedocs.io/en/latest/replica_masking/ +[Masking Views]: https://postgresql-anonymizer.readthedocs.io/en/stable/masking_views/ +[Masking Data Wrappers]: https://postgresql-anonymizer.readthedocs.io/en/stable/masking_data_wrappers/ +[INSTALL]: https://postgresql-anonymizer.readthedocs.io/en/latest/INSTALL/ + + +Introducing Replica Masking +-------------------------------------------------------------------------------- + +In some situations, you may want to have an anonymized copy of your production +database on another instance like with Backup Masking (aka "Anonymized Dumps") +but you also would like this copy to be up-to-date with the original data like +with Dynamic Masking… + +With the Replica Masking feature, you can use PostgreSQL logical replication to +create an anonymized clone of your production database. + +See the [documentation](https://postgresql-anonymizer.readthedocs.io/en/latest/replica_masking/) +for more details. + +/!\ WARNING! DO NOT USE IN PRODUCTION + +This feature is currently under heavy development. This implementation of +Replica Masking is provided for testing purpose only. Major breaking changes +may be introduced at any time and we may even remove this feature entirely if +we feel it does not reach our standard of quality and stability. + +We welcome any feedback, testing reports, comments and contributions. But for +the moment, we do not guarantee any form of support for this feature. + +Our current plan is to stabilize this feature in version 3.0, which is scheduled +for early 2026. + + +Acknowledgments +-------------------------------------------------------------------------------- + +This release also includes code, bugfixes, documentation, code reviews and ideas +from Robin Portigliatti, Suhas Thalanki, Benoit Lobréau and other [contributors]. + +And also special thanks to the [PGRX] team for their amazing work! + +[contributors]: https://gitlab.com/dalibo/postgresql_anonymizer/-/blob/master/AUTHORS.md +[PGRX]: https://github.com/pgcentralfoundation/pgrx + + +Join our community to improve data privacy! +-------------------------------------------------------------------------------- + +PostgreSQL Anonymizer is part of the [Dalibo Labs] initiative. It is mainly +developed by [Damien Clochard]. + +This is an open project, contributions are welcome. We need your feedback and +ideas! Let us know what you think of this tool, how it fits your needs and +what features are missing. + +If you want to help, you can find a [list of `Junior Jobs`](https://gitlab.com/dalibo/postgresql_anonymizer/issues?label_name%5B%5D=Junior+Jobs). + + +-------------------------------------------------------------------------------- + PostgreSQL Anonymizer 2.2: Masking Cursors ================================================================================ @@ -158,7 +259,7 @@ on-the-fly with [Dynamic Masking] and [Anonymous Dumps], or permanently with Acknowledgments -------------------------------------------------------------------------------- -The image blurring feature was developped by Pierre-Marie Petit. Kudos to him +The image blurring feature was developed by Pierre-Marie Petit. Kudos to him for this tremendous innovation! This release also includes code, bugfixes, documentation, code reviews and ideas diff --git a/anon.control b/anon.control index bb00cda..6b7dd80 100644 --- a/anon.control +++ b/anon.control @@ -4,4 +4,5 @@ default_version = '@CARGO_VERSION@' #directory='extension/anon' relocatable = false superuser = true +trusted = true module_pathname = '$libdir/anon' diff --git a/docs/images/anon-Anonymized_Replica.drawio.png b/docs/images/anon-Anonymized_Replica.drawio.png new file mode 100644 index 0000000..0b5dd56 Binary files /dev/null and b/docs/images/anon-Anonymized_Replica.drawio.png differ diff --git a/docs/images/anon-Anonymized_Standby.drawio.png b/docs/images/anon-Anonymized_Standby.drawio.png new file mode 100644 index 0000000..e7f1c75 Binary files /dev/null and b/docs/images/anon-Anonymized_Standby.drawio.png differ diff --git a/docs/images/anon-Dump.drawio.png b/docs/images/anon-Dump.drawio.png index a93d997..3817fc6 100644 Binary files a/docs/images/anon-Dump.drawio.png and b/docs/images/anon-Dump.drawio.png differ diff --git a/docs/replica_masking.md b/docs/replica_masking.md new file mode 100644 index 0000000..1bab986 --- /dev/null +++ b/docs/replica_masking.md @@ -0,0 +1,247 @@ +Anonymous Replica +=============================================================================== + +WARNING! DO NOT USE IN PRODUCTION +------------------------------------------------------------------------------- + +This feature is currently under heavy development. This implementation of +Replica Masking is provided for testing purpose only. Major breaking changes +may be introduced at any time and we may even remove this feature entirely +if we feel it does not reach our standard of quality and stability. + +We welcome any feedback, testing reports, comments and contributions. But at +the moment, we do not guarantee any form of support for this feature. + +Our current plan is to stabilize this feature in version 3.0, which is +scheduled for early 2026. + +Thanks for your understanding. + +Principle +------------------------------------------------------------------------------- + +In some situations, you may want to have an anonymized copy of your +production database on another instance like with [Backup Masking] +(aka "Anonymized Dumps") but you also would like this copy to be up-to-date +with the original data like with [Dynamic Masking]... + +With the Replica Masking feature, you can use PostgreSQL logical replication +to create an anonymized clone of your production database. + +![PostgreSQL Replica Masking](images/anon-Anonymized_Replica.drawio.png) + +[Backup Masking]: anonymous_dumps.md +[Dynamic Masking]: dynamic_masking.md + + +Preamble: Learn about logical replication ! +------------------------------------------------------------------------------- + +PostgreSQL logical replication is a powerful mechanism. Before setting up a +anonymous replica, be sure that you are able to configure standard logical +replication correctly. + +There are many tutorials available for that and we also recommend reading the +PostgreSQL manual: + + + + +Quick Setup +------------------------------------------------------------------------------- + +### Example + +Let's say we want to anonymize a table `person` in a database `foo` like this: + +```sql +CREATE TABLE person ( + id SERIAL PRIMARY KEY, + name TEXT, + company TEXT +); + + +INSERT INTO person VALUES (1, 'Alice', 'CompanyA'); +INSERT INTO person VALUES (2, 'Bob', 'CompanyB'); +INSERT INTO person VALUES (3, 'Charlie', 'CompanyC'); +INSERT INTO person VALUES (4, 'David', 'CompanyD'); +INSERT INTO person VALUES (5, 'Eve', 'CompanyE'); + +``` + +### A- On the publisher database + +A1- Create a replication role: + +```sql +CREATE ROLE anon_replicator LOGIN REPLICATION PASSWORD 'CHANGE-ME-3747'; +GRANT USAGE ON SCHEMA public TO anon_replicator; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO anon_replicator; +``` + +Be sure to configure your `pg_hba.conf` file to allow `anon_replicator` to +connect from the subscriber database. + + +A2- Create a publication: + +```sql +CREATE PUBLICATION pub FOR TABLE person; +``` + +All of this is pretty standard. There's nothing special regarding anonymization +on the publisher database. In fact, the publisher database “does not know” +that the data will be masked on the subscriber. + +### B- On the subscriber database + +B1- Create the table ( DDL commands are NOT replicated ): + +```sql +CREATE TABLE person ( + id SERIAL PRIMARY KEY, + name TEXT, + company TEXT +); + +``` + +B2- Enable replica masking: + +```sql +ALTER DATABASE foo SET anon.replica_masking TO on; +``` + +B3- Reconnect to the database so that the configuration is applied. + +B4- Define the masking rules: + +```sql +SECURITY LABEL FOR anon ON COLUMN person.company + IS 'MASKED WITH FUNCTION pg_catalog.md5(company)'; + +SECURITY LABEL FOR anon ON COLUMN person.name + IS 'MASKED WITH FUNCTION anon.dummy_first_name()'; +``` + +B5- start the replica masking engine: + +```sql +SELECT anon.start_replica_masking(); +``` + +B6- Create the subscription: + +```sql +CREATE SUBSCRIPTION anon_sub +CONNECTION 'host=prod_srv user=anon_replicator password=CHANGE-ME-3747 dbname=foo' +PUBLICATION pub; +``` + +Wait for a few milliseconds while the data is being synchronized and masked... + +Et voilà ! + + +```sql +SELECT * FROM person; + + id | name | company +----+-----------+---------------------------------- + 1 | Christine | a1e551387ba94e882ccc5356948d6462 + 2 | Percival | 75b4e152a05dae2f1d7991182e707fad + 3 | Ignatius | e2a211f97064ee5a86853ae61e1bb2b9 + 4 | Karley | 8d543957c23828bb0d888cf7da59a817 + 5 | Alfredo | 566ca1969819cbf2098202255914bf23 +``` + +Changing the masking rules +------------------------------------------------------------------------------- + +Anytime you add or remove a masking rule, you need to update the replica masking +engine. + +```sql +SELECT anon.refresh_replica_masking(); +``` + +Anonymized Standby +------------------------------------------------------------------------------- + +In complement to Replica Masking, it is possible to use Hot Standby replication +to build a distant clone of the Anonymized Replica. This is useful to export +the database to a remote datacenter because the Anonymized Replica will operate +as a masking proxy, "cleaning" the personal information before it gets +transferred to the Standby instance. + +![PostgreSQL Standby Masking](images/anon-Anonymized_Standby.drawio.png) + + +Security +------------------------------------------------------------------------------- + +Keep in mind that the masking rules are applied on-the-fly in the subscriber +database, which means: + +* The original data is transferred through the connection between the publisher + and the subscriber. Therefore this connection should be protected like + in a regular logical replication setup. + +* The superuser of the subscriber instance and the owner of the subscriber + database can disable Replica Masking at anytime. They can both access the + original, just like the superuser and the owner of the publisher database. + Therefore, a third role should be created on the subscriber database to provide + unprivileged and read-only access to the data. + +* The replication role is also able to access the original data at any time. + +* The logs of the subscriber database may contain unmasked data. + + +Limitations +------------------------------------------------------------------------------- + +* Anonymous replication is based on logical replication, therefore it has the + same [restrictions], in particular: DDL commands, sequences, Large Objects + are NOT replicated. + +* The `REPLICA IDENTITY FULL` method is NOT supported. This means that all + replicated tables MUST have a primary key. + +* The primary key of a table should not be masked. + +[restrictions]: https://www.postgresql.org/docs/current/logical-replication-restrictions.html + + +But I want to anonymize a primary key! +------------------------------------------------------------------------------- + +If you need to anonymize a primary key in a table, this means that it is a +natural key (as opposed to a [surrogate key]). + +[surrogate key]: https://en.wikipedia.org/wiki/Surrogate_key + +Natural keys are problematic for many reasons: + +* they can change over time (like email addresses or product codes), forcing + cascading updates throughout related tables +* they're often not truly unique in practice, even seemingly unique values + like SSNs can have duplicates or exceptions +* they tend to be longer and more complex than simple integers +* they make joins slower and indexes larger +* they can contain sensitive information that you might not want exposed in + URLs or logs. +* they may change whenever business rules evolve, requiring database + restructuring. + +Surrogate keys (i.e. auto-incrementing integers) avoid these issues by +providing stable, meaningless identifiers that never need to change. + +In particular for anonymization: surrogate keys make your life easier since you +don't have to mask them. +In the other hand, natural keys are often a nightmare: in most situations they +will force you to use complex pseudonymization techniques, and keep in mind that +that [Pseudonymization Is Not Anonymization] ! + +[Pseudonymization Is Not Anonymization]: masking_functions.md#pseudonymization diff --git a/docs/runbooks/0-intro.md b/docs/runbooks/0-intro.md index 3d3cbb0..17a9f36 100644 --- a/docs/runbooks/0-intro.md +++ b/docs/runbooks/0-intro.md @@ -66,6 +66,10 @@ In order to make this workshop, you will need: - A role "pierre" and a role "jack", both allowed to connect to the database "boutique" +Check out the [INSTALL] section to learn how to install +the [PostgreSQL Anonymizer] extension: + +[INSTALL]: https://postgresql-anonymizer.readthedocs.io/en/stable/INSTALL/ !!! tip diff --git a/docs/runbooks/1-static_masking.md b/docs/runbooks/1-static_masking.md index 327db63..141d4cb 100644 --- a/docs/runbooks/1-static_masking.md +++ b/docs/runbooks/1-static_masking.md @@ -5,11 +5,17 @@ run-sql: - parse_query: False ... -# 1 - Static Masking +# 1- Static Masking -> Static Masking is the simplest way to hide personal information! This -> idea is simply to destroy the original data or replace it with an -> artificial one. +💡 Static Masking is the simplest way to hide personal information! +This idea is simply to destroy the original data or replace it with +an artificial one. + +## Requirements + +**Please check out the [intro] of this tutorial if you haven't read it yet** + +[intro]: tutorials/0-intro/ ## The story @@ -139,7 +145,7 @@ again. Paul realizes that the postcode gives a clear indication of where his customers live. However he would like to have statistics based on their -`postcode area`. +postcode area. **Add a new masking rule to replace the last 3 digits by 'x'.** @@ -155,9 +161,9 @@ date of the customers. Replace all the birth dates by January 1rst, while keeping the real year. -!!! hint +💡 You can use the [make_date] or [date_trunc] functions ! - You can use the [make_date] or [date_trunc] functions ! +See [make_date]: https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-TABLE [date_trunc]: https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-TABLE @@ -182,9 +188,7 @@ FROM customer c JOIN best_client b ON (c.id = b.fk_customer_id) ``` -!!! note - - This is called **[Singling Out] a person.** +💡 This is called **[Singling Out] a person.** [Singling Out]: https://www.pnas.org/content/117/15/8344 @@ -203,12 +207,10 @@ the integrity of the data? Find a function that will shuffle the column `fk_company_id` of the `payout` table -!!! tip - - Check out the [static masking] section of the [documentation]. +💡 Check out the [shuffling] section of the [documentation]. -[shuffling]: https://postgresql-anonymizer.readthedocs.io/en/stable/static_masking#shuffling +[shuffling]: static_masking#shuffling [documentation]: https://postgresql-anonymizer.readthedocs.io/en/stable/ ## Solutions diff --git a/docs/runbooks/2-dynamic_masking.md b/docs/runbooks/2-dynamic_masking.md index de5e310..aa67194 100644 --- a/docs/runbooks/2-dynamic_masking.md +++ b/docs/runbooks/2-dynamic_masking.md @@ -3,37 +3,49 @@ run-sql: - dbname: boutique - user: paul - parse_query: False + ... -# 2- How to use Dynamic Masking +2- Dynamic Masking +=============================================================================== + +💡 With Dynamic Masking, the database owner can hide personal data for +some users, while other users are still allowed to read and write the +authentic data. + +Requirements +------------------------------------------------------------------------------- -> With Dynamic Masking, the database owner can hide personal data for -> some users, while other users are still allowed to read and write the -> authentic data. +**Please check out the [intro] of this tutorial if you haven't read it yet** -## The Story +[intro]: tutorials/0-intro/ + +The Story +------------------------------------------------------------------------------- Paul has 2 employees: -- Jack is operating the new sales application, he needs access to the - real data. He is what the GPDR would call a **\"data processor\"**. -- Pierre is a data analyst who runs statistic queries on the database. - He should not have access to any personal data. +- Jack is operating the new sales application, he needs access to the + real data. He is what the GPDR would call a **\"data processor\"**. +- Pierre is a data analyst who runs statistic queries on the database. + He should not have access to any personal data. -## How it works +How it works +------------------------------------------------------------------------------- ![](../images/anon-Dynamic.drawio.png) -## Objectives +Objectives +------------------------------------------------------------------------------- In this section, we will learn: -- How to write simple masking rules -- The advantage and limitations of dynamic masking -- The concept of \"Linkability\" of a person - -## The `company` table +- How to write simple masking rules +- The advantage and limitations of dynamic masking +- The concept of \"Linkability\" of a person +The `company` table +------------------------------------------------------------------------------- ``` { .run-postgres parse_query=False } @@ -61,7 +73,8 @@ VALUES SELECT * FROM company; ``` -## The `supplier` table +The `supplier` table +------------------------------------------------------------------------------- ``` { .run-postgres parse_query=False } CREATE TABLE supplier ( @@ -85,7 +98,8 @@ VALUES SELECT * FROM supplier; ``` -## Activate the extension +Activate the extension +------------------------------------------------------------------------------- ``` run-postgres ALTER DATABASE boutique @@ -96,7 +110,8 @@ CREATE EXTENSION IF NOT EXISTS anon; SELECT anon.init(); ``` -## Dynamic Masking +Dynamic Masking +------------------------------------------------------------------------------- ### Activate the masking engine @@ -110,6 +125,7 @@ ALTER DATABASE boutique ``` run-postgres SECURITY LABEL FOR anon ON ROLE pierre IS 'MASKED'; + GRANT pg_read_all_data to pierre; ``` @@ -122,7 +138,8 @@ SELECT * FROM supplier; For the moment, there is no masking rule so Pierre can see the original data in each table. -## Masking the supplier names +Masking the supplier names +------------------------------------------------------------------------------- Connect as Paul and define a masking rule on the supplier table: @@ -131,8 +148,7 @@ SECURITY LABEL FOR anon ON COLUMN supplier.contact IS 'MASKED WITH VALUE $$CONFIDENTIAL$$'; ``` - ------------------------------------------------------------------------- +--- Now connect as Pierre and try to read the supplier table again: @@ -140,7 +156,7 @@ Now connect as Pierre and try to read the supplier table again: SELECT * FROM supplier; ``` ------------------------------------------------------------------------- +--- Now connect as Jack and try to read the real data: @@ -148,21 +164,24 @@ Now connect as Jack and try to read the real data: SELECT * FROM supplier; ``` -## Exercises +Exercises +------------------------------------------------------------------------------- ### E201 - Guess who is the CEO of "Johnny's Shoe Store" -Masking the supplier name is clearly not enough to provide anonymity. +Masking the supplier contact is clearly not enough to provide anonymity. -**Connect as Pierre and write a simple SQL query that would reindentify -some suppliers based on their job and their company.** +**Connect as Pierre and write a simple SQL query that joins the `supplier` +and the `company` tables. See how that could reindentify some suppliers +based on their job and their company.** -Company names and job positions are available in many public datasets. A -simple search on Linkedin or Google, would give you the names of the top -executives of most companies.. +With this request we managed to link a person to a company and we know +it's job title. Since company names and job positions are available in +many public datasets: a simple search on Linkedin or Google would give +us the real names of many of the employees of these companies... -> This is called **Linkability**: the ability to connect multiple -> records concerning the same data subject. +💡 This is called **Linkability**: the ability to connect multiple +records concerning the same data subject. ### E202 - Anonymize the companies @@ -170,21 +189,21 @@ We need to anonymize the `company` table, too. Even if they don't contain personal information, some fields can be used to **infer** the identity of their employees... -**Write 2 masking rules for the company table. The first one will -replace the `name` field with a fake name. The second will replace the -`vat_id` with a random sequence of 10 characters** +**Connect as Paul and write 2 masking rules (security labels) for the company table.** -!!! tip +* The first one will replace the `name` field with a fake name. +* The second rule will replace the `vat_id` with a random sequence of 10 characters - Go to the[documentation] and look at the [faking functions] and the - [random functions] ! +💡 Go to the[documentation] and look at the [faking functions] and the +[random functions] ! [documentation]: https://postgresql-anonymizer.readthedocs.io/en/stable/ -[faking functions]: https://postgresql-anonymizer.readthedocs.io/en/stable/masking_functions#faking -[random functions]: https://postgresql-anonymizer.readthedocs.io/en/stable/masking_functions#randomization +[faking functions]: masking_functions#faking +[random functions]: masking_functions#randomization Connect as Pierre and check that he cannot view the real company info. +Connect as Jack and check that he can view the real values. ### E203 - Pseudonymize the company name @@ -192,17 +211,21 @@ Because of dynamic masking, the fake values will be different every time Pierre tries to read the table. Pierre would like to have always the same fake values for a given -company. **This is called pseudonymization.** +company. + +💡 **This is called pseudonymization.** -**Write a new masking rule over the `vat_id` field by generating 10 -random characters using the md5() function.** +**Connect as Paul and write a new masking rule over the `vat_id` +field by generating a hash of 10 characters using the `anon.digest()` +function.** **Write a new masking rule over the `name` field by using a [pseudonymizing function].** -[pseudonymizing function]: https://postgresql-anonymizer.readthedocs.io/en/stable/masking_functions#pseudonymization +[pseudonymizing function]: masking_functions#pseudonymization -## Solutions +Solutions +------------------------------------------------------------------------------- ### S201 @@ -235,8 +258,20 @@ Pierre will see different "fake data" every time he reads the table: SELECT * FROM company; ``` +Jack still sees the real data + +``` { .run-postgres user=jack } +SELECT * FROM company; +``` + + ### S203 +``` run-postgres +SECURITY LABEL FOR anon ON COLUMN company.vat_id +IS $$ MASKED WITH FUNCTION anon.left(anon.digest(vat_id, 'xxx', 'md5'),10) $$; +``` + ``` run-postgres SECURITY LABEL FOR anon ON COLUMN company.name IS 'MASKED WITH FUNCTION anon.pseudo_company(id)'; diff --git a/docs/runbooks/3-anonymous_dumps.md b/docs/runbooks/3-anonymous_dumps.md index 2c675c8..6140dfd 100644 --- a/docs/runbooks/3-anonymous_dumps.md +++ b/docs/runbooks/3-anonymous_dumps.md @@ -2,16 +2,17 @@ run-sql: - dbname: boutique - user: paul - - parse_query: False ... -# 3- Anonymous Dumps +3- Anonymous Dumps +=============================================================================== -> In many situation, what we want is basically to export the anonymized -> data into another database (for testing or to produce statistics). -> We will simply use pg_dump for that ! +💡 In many situation, what we want is basically to export the anonymized +data into another database (for testing or to produce statistics). +We will simply use pg_dump for that ! -## The Story +The Story +------------------------------------------------------------------------------- Paul has a website and a comment section where customers can express their views. @@ -21,16 +22,19 @@ agency asked for a SQL export (dump) of the current website database. Paul wants to `clean` the database export and remove any personal information contained in the comment section. -## How it works +How it works +------------------------------------------------------------------------------- ![](../images/anon-Dump.drawio.png) -## Learning Objective +Learning Objective +------------------------------------------------------------------------------- -- Extract the anonymized data from the database -- Write a custom masking function to handle a JSON field. +- Extract the anonymized data from the database +- Write a custom masking function to handle a JSON field. -## Load the data +Load the data +------------------------------------------------------------------------------- ``` run-postgres DROP TABLE IF EXISTS website_comment CASCADE; @@ -68,16 +72,18 @@ SELECT message->'meta'->'name' AS name, message->'content' AS content FROM website_comment -ORDER BY id ASC +ORDER BY id ASC; ``` -## Activate the extension +Activate the extension +------------------------------------------------------------------------------- ``` run-postgres CREATE EXTENSION IF NOT EXISTS anon; ``` -## Masking a JSON column +Masking a JSON column +------------------------------------------------------------------------------- The `comment` field is filled with personal information and the fact the field does not have a standard schema makes our tasks harder. @@ -88,19 +94,18 @@ As we can see, web visitors can write any kind of information in the comment section. Our best option is to remove this key entirely because there's no way to extract personal data properly. - ------------------------------------------------------------------------- +--- We can *clean* the comment column simply by removing the `content` -key! +key in the `message` column ! ``` run-postgres -SELECT message - ARRAY['content'] +SELECT message - ARRAY['content'] AS message_without_content FROM website_comment WHERE id=1; ``` ------------------------------------------------------------------------- +--- First let's create a dedicated schema and declare it as trusted. This means the `anon` extension will accept the functions located in this @@ -114,8 +119,7 @@ CREATE SCHEMA IF NOT EXISTS my_masks; SECURITY LABEL FOR anon ON SCHEMA my_masks IS 'TRUSTED'; ``` - ------------------------------------------------------------------------- +--- Now we can write a function that remove the message content: @@ -129,14 +133,13 @@ LANGUAGE SQL ; ``` - ------------------------------------------------------------------------- +--- Let's try it! ``` run-postgres SELECT my_masks.remove_content(message) -FROM website_comment +FROM website_comment; ``` @@ -179,32 +182,40 @@ export PGHOST=localhost pg_dump -U anon_dumper boutique --table=website_comment > /tmp/dump.sql ``` -## Exercises +Exercises +------------------------------------------------------------------------------- ### E301 - Dump the anonymized data into a new database Create a database named `boutique_anon` and transfer the entire database into it. -### E302 - Pseudonymize the meta fields of the comments +### E302 - Remove the email address + + +Replace the `remove_content` function with a better one called +`remove_content_and_ip` that will nullify the `email` key. + +💡 HINT: you can use `jsonb_set(message, '{meta, email}', '{}')` +to remove the email value. + + +### E303 - Pseudonymize the IP address Pierre plans to extract general information from the metadata. For instance, he wants to calculate the number of unique visitors based on -the different IP addresses. But an IP address is an **indirect -identifier**, so Paul needs to anonymize this field while maintaining -the fact that some values appear multiple times. +the different IP addresses. -Replace the `remove_content` function with a better one called -`clean_comment` that will: +But an IP address is an **indirect identifier**, so Paul needs to anonymize +this field while maintaining the fact that some values appear multiple times. -- Remove the content key -- Replace the `name` value with a fake last name -- Replace the `ip_address` value with its MD5 signature -- Nullify the `email` key -> HINT: Look at the `jsonb_set()` and `jsonb_build_object()` functions +💡 HINT: First you can create a new `meta` object using `jsonb_build_object()` +and then use function `jsonb_set` replace the `meta` key -## Solutions + +Solutions +------------------------------------------------------------------------------- ### S301 @@ -223,6 +234,30 @@ psql -U paul boutique_anon -c 'SELECT COUNT(*) FROM company' ### S302 +```run-postgres +CREATE OR REPLACE FUNCTION my_masks.remove_content_and_ip(message JSONB) +RETURNS JSONB +VOLATILE +LANGUAGE SQL +AS $func$ +SELECT + jsonb_set(message, '{meta, email}', '{}') + - ARRAY['content']; +$func$; +``` + +``` run-postgres +SELECT my_masks.remove_content_and_ip(message) +FROM website_comment; +``` + +``` run-postgres +SECURITY LABEL FOR anon ON COLUMN website_comment.message +IS 'MASKED WITH FUNCTION my_masks.remove_content_and_ip(message)'; +``` + +### S303 + ```run-postgres CREATE OR REPLACE FUNCTION my_masks.clean_comment(message JSONB) RETURNS JSONB diff --git a/docs/runbooks/4-generalization.md b/docs/runbooks/4-generalization.md index 23b4b48..42f4980 100644 --- a/docs/runbooks/4-generalization.md +++ b/docs/runbooks/4-generalization.md @@ -5,14 +5,13 @@ run-sql: - parse_query: False ... -# 4 - Generalization +# 4- Generalization - -> The main idea of generalization is to `blur` the original data. For -> example, instead of saying `Mister X was born on July 25, 1989`, we -> can say `Mister X was born is the 80's`. The information is still -> true, but it is less precise and it can\'t be used to reidentify the -> subject. +💡 The main idea of generalization is to `blur` the original data. For +example, instead of saying `Mister X was born on July 25, 1989`, we +can say `Mister X was born is the 80's`. The information is still +true, but it is less precise and it can't be used to reidentify the +subject. ## The Story @@ -52,11 +51,7 @@ CREATE TABLE employee ( ); ``` - - -!!! danger - This is awkward and illegal. - +🚨 This is awkward and illegal. Loading the data: @@ -112,12 +107,20 @@ FROM v_asthma_eyes GROUP BY eyes; ``` -Pierre just proved that asthma is caused by green eyes. +Pierre just proved that asthma is caused by blue eyes ;-) ## K-Anonymity The `asthma` and `eyes` columns are considered as indirect identifiers. +Indirect personal identifiers (or +"quasi-identifiers") are pieces of information that, when combined with +other data can identify an individual. Examples of indirect identifiers +include: Date of birth, Gender, Zip code, etc. + +With PostgreSQL Anonymizer, we can declare that a column is an indirect +identifiers, like this: + ``` run-postgres SECURITY LABEL FOR k_anonymity ON COLUMN v_asthma_eyes.eyes @@ -132,8 +135,8 @@ SECURITY LABEL FOR k_anonymity SELECT anon.k_anonymity('v_asthma_eyes'); ``` -The v_asthma_eyes has \'2-anonymity\'. This means that each -quasi-identifier combination (the \'eyes-asthma\' tuples) occurs in at +The v_asthma_eyes has '2-anonymity'. This means that each +quasi-identifier combination (the 'eyes-asthma' tuples) occurs in at least 2 records for a dataset. In other words, it means that each individual in the view cannot be @@ -141,6 +144,10 @@ distinguished from at least 1 (k-1) other individual. ## Range and Generalization functions +Now let's add another view over the `employee` table. + +We will generalize the dates of to keep only the month and year. + ``` run-postgres DROP MATERIALIZED VIEW IF EXISTS v_staff_per_month; CREATE MATERIALIZED VIEW v_staff_per_month AS @@ -172,7 +179,8 @@ FROM v_staff_per_month; ### Declaring the indirect identifiers Now let's check the k-anonymity of this view by declaring which columns -are indirect identifiers. +are indirect identifiers : + ``` run-postgres SECURITY LABEL FOR k_anonymity @@ -186,8 +194,9 @@ SECURITY LABEL FOR k_anonymity SELECT anon.k_anonymity('v_staff_per_month'); ``` -In this case, the k factor is 1 which means that at least one unique -individual can be identified directly by his/her first and last dates. +In this case, the k factor is 1 which means that there is at least one +unique individual who be identified directly by his/her first and last +dates. ## Exercises @@ -224,8 +233,7 @@ SELECT FROM employee; ``` -!!! tip - '[]' will include the upper bound +💡 `'[]'` will include the upper bound ---- diff --git a/docs/runbooks/9-conclusion.md b/docs/runbooks/9-conclusion.md index 3a74d01..dd87cde 100644 --- a/docs/runbooks/9-conclusion.md +++ b/docs/runbooks/9-conclusion.md @@ -1,9 +1,11 @@ -# Conclusion +Conclusion +=============================================================================== ---- -## Clean up ! +Clean up ! +------------------------------------------------------------------------------- ``` { .run-postgres user=postgres dbname=postgres } DROP DATABASE IF EXISTS boutique; @@ -28,14 +30,16 @@ DROP ROLE IF EXISTS pierre; DROP ROLE IF EXISTS dump_anon; ``` -## Also... +Also... +------------------------------------------------------------------------------- Other projects you may like -- [pg_sample](https://github.com/mla/pg_sample) : extract a small - dataset from a larger PostgreSQL database +- [pg_sample](https://github.com/mla/pg_sample) : extract a small + dataset from a larger PostgreSQL database -## Help Wanted! +Help Wanted! +------------------------------------------------------------------------------- This is a free and open project! @@ -43,4 +47,3 @@ This is a free and open project! Please send us feedback on how you use it, how it fits your needs (or not), etc. - diff --git a/docs/sampling.md b/docs/sampling.md index 5f0a81c..5cd4519 100644 --- a/docs/sampling.md +++ b/docs/sampling.md @@ -25,6 +25,10 @@ With PostgreSQL Anonymizer, you can use 2 different sampling methods : * [Sampling with TABLESAMPLE](#sampling_with_tablesample) * [Sampling with RLS Policies](#sampling_with_rls_policies) +You can also [Truncate Tables for the masked users] ! + +[Truncate Tables for the masked users]: #truncate-tables-for-the-masked-users + Sampling with TABLESAMPLE ------------------------------------------------------------------------------- @@ -137,3 +141,47 @@ There may be other sampling tools for PostgreSQL but [pg_sample] is probably the best one. [pg_sample]: https://github.com/mla/pg_sample + + +Truncate Tables for the masked users +------------------------------------------------------------------------------- + +In certain situations, you can also erase complety a table instead of just +masking some of the columns. + +For instance, let's say that masked users should not see anything in the +`http_logs` table below + +```sql +CREATE TABLE http_logs ( + id integer NOT NULL, + date_opened DATE, + ip_address INET, + url TEXT +); +``` + +Using the [TABLESAMPLE clause], you can simply set the sampling ratio to 0 + + +```sql +SECURITY LABEL FOR anon ON TABLE http_logs IS ' TABLESAMPLE SYSTEM (0)'; +``` + +Now the table will be erased for the masked users ! + +```sql +SET ROLE the_database_owner; + +SELECT count(*) FROM http_logs; + count +--------- + 156706 + +SET ROLE a_masked_user; + +SELECT count(*) FROM http_logs; + count +--------- + 0 +``` diff --git a/docs/tutorials/0-intro.md b/docs/tutorials/0-intro.md index c1e3ea7..84a9a25 100644 --- a/docs/tutorials/0-intro.md +++ b/docs/tutorials/0-intro.md @@ -26,9 +26,9 @@ data. Using the simple example above, we will learn: -- How to write masking rules -- The difference between static and dynamic masking -- Implementing advanced masking techniques +- How to write masking rules +- The difference between static and dynamic masking +- Implementing advanced masking techniques ## About GDPR @@ -37,23 +37,28 @@ general concepts of anonymization. For more information about it, please refer to the talk below: -- [Anonymisation, Au-delà du - RGPD](https://www.youtube.com/watch?v=KGSlp4UygdU) (Video / French) -- [Anonymization, Beyond - GDPR](https://public.dalibo.com/exports/conferences/_archives/_2019/20191016_anonymisation_beyond_GDPR/anonymisation_beyond_gdpr.pdf) - (PDF / english) +- [Anonymisation, Au-delà du + RGPD](https://www.youtube.com/watch?v=KGSlp4UygdU) (Video / French) +- [Anonymization, Beyond + GDPR](https://public.dalibo.com/exports/conferences/_archives/_2019/20191016_anonymisation_beyond_GDPR/anonymisation_beyond_gdpr.pdf) + (PDF / english) ## Requirements In order to make this workshop, you will need: -- A Linux VM ( preferably `Debian 12 bookworm` or `Ubuntu 24.04`) -- A PostgreSQL instance ( preferably `PostgreSQL 17` ) -- The PostgreSQL Anonymizer (anon) extension, installed and - initialized by a superuser -- A database named "boutique" owned by a **superuser** called "paul" -- A role "pierre" and a role "jack", both allowed to connect to the - database "boutique" +- A Linux VM ( preferably `Debian 12 bookworm` or `Ubuntu 24.04`) +- A PostgreSQL instance ( preferably `PostgreSQL 17` ) +- The PostgreSQL Anonymizer (anon) extension, installed and initialized + by a superuser +- A database named "boutique" owned by a **superuser** called "paul" +- A role "pierre" and a role "jack", both allowed to connect to the + database "boutique" + +Check out the +[INSTALL](https://postgresql-anonymizer.readthedocs.io/en/stable/INSTALL/) +section to learn how to install the [PostgreSQL +Anonymizer](https://labs.dalibo.com/postgresql_anonymizer) extension: !!! tip @@ -82,17 +87,13 @@ to learn how to install the extension in your PostgreSQL instance. We will with 3 different users: -``` sql +``` {.sql user="postgres" dbname="postgres" show_result="false"} CREATE ROLE paul LOGIN SUPERUSER PASSWORD 'CHANGEME'; - CREATE ROLE pierre LOGIN PASSWORD 'CHANGEME'; - CREATE ROLE jack LOGIN PASSWORD 'CHANGEME'; - GRANT pg_read_all_data TO jack; - GRANT pg_write_all_data TO jack; ``` @@ -114,13 +115,13 @@ chmod 0600 ~/.pgpass We will work on a database called "boutique": -``` sql +``` {.sql user="postgres" dbname="postgres"} CREATE DATABASE boutique OWNER paul; ``` We need to activate the `anon` library inside that database: -``` sql +``` {.sql user="postgres" dbname="postgres"} ALTER DATABASE boutique -SET session_preload_libraries = 'anon'; + SET session_preload_libraries = 'anon'; ``` diff --git a/docs/tutorials/1-static_masking.md b/docs/tutorials/1-static_masking.md index c5835b9..cfa424d 100644 --- a/docs/tutorials/1-static_masking.md +++ b/docs/tutorials/1-static_masking.md @@ -1,8 +1,13 @@ -# 1 - Static Masking +# 1- Static Masking -> Static Masking is the simplest way to hide personal information! This -> idea is simply to destroy the original data or replace it with an -> artificial one. +💡 Static Masking is the simplest way to hide personal information! This +idea is simply to destroy the original data or replace it with an +artificial one. + +## Requirements + +**Please check out the [intro](tutorials/0-intro/) of this tutorial if +you haven't read it yet** ## The story @@ -20,30 +25,40 @@ it. In this section, we will learn: -- How to write simple masking rules -- The advantage and limitations of static masking -- The concept of "Singling Out" a person +- How to write simple masking rules +- The advantage and limitations of static masking +- The concept of "Singling Out" a person ## The "customer" table -``` sql +``` {.sql parse_query="False"} DROP TABLE IF EXISTS customer CASCADE; + DROP TABLE IF EXISTS payout CASCADE; -CREATE TABLE customer ( id SERIAL PRIMARY KEY, firstname TEXT, lastname TEXT, phone TEXT, birth DATE, postcode TEXT ); + +CREATE TABLE customer ( + id SERIAL PRIMARY KEY, + firstname TEXT, + lastname TEXT, + phone TEXT, + birth DATE, + postcode TEXT +); ``` Insert a few persons: ``` sql INSERT INTO customer -VALUES (107,'Sarah','Conor','060-911-0911', '1965-10-10', '90016'), - (258,'Luke', 'Skywalker', NULL, '1951-09-25', '90120'), - (341,'Don', 'Draper','347-515-3423', '1926-06-01', '04520') ; +VALUES +(107,'Sarah','Conor','060-911-0911', '1965-10-10', '90016'), +(258,'Luke', 'Skywalker', NULL, '1951-09-25', '90120'), +(341,'Don', 'Draper','347-515-3423', '1926-06-01', '04520') +; ``` ``` sql -SELECT * -FROM customer; +SELECT * FROM customer; ``` | id | firstname | lastname | phone | birth | postcode | @@ -56,19 +71,27 @@ FROM customer; Sales are tracked in a simple table: -``` sql -CREATE TABLE payout ( id SERIAL PRIMARY KEY, fk_customer_id INT REFERENCES customer(id), order_date DATE, payment_date DATE, amount INT ); +``` {.sql parse_query="False"} +CREATE TABLE payout ( + id SERIAL PRIMARY KEY, + fk_customer_id INT REFERENCES customer(id), + order_date DATE, + payment_date DATE, + amount INT +); ``` Let's add some orders: ``` sql INSERT INTO payout -VALUES (1,107,'2021-10-01','2021-10-01', '7'), - (2,258,'2021-10-02','2021-10-03', '20'), - (3,341,'2021-10-02','2021-10-02', '543'), - (4,258,'2021-10-05','2021-10-05', '12'), - (5,258,'2021-10-06','2021-10-06', '92') ; +VALUES +(1,107,'2021-10-01','2021-10-01', '7'), +(2,258,'2021-10-02','2021-10-03', '20'), +(3,341,'2021-10-02','2021-10-02', '543'), +(4,258,'2021-10-05','2021-10-05', '12'), +(5,258,'2021-10-06','2021-10-06', '92') +; ``` ## Activate the extension @@ -82,9 +105,12 @@ CREATE EXTENSION IF NOT EXISTS anon; Paul wants to hide the last name and the phone numbers of his clients. He will use the `dummy_last_name()` and `partial()` functions for that: -``` sql -SECURITY LABEL FOR anon ON COLUMN customer.lastname IS 'MASKED WITH FUNCTION anon.dummy_last_name()'; -SECURITY LABEL FOR anon ON COLUMN customer.phone IS 'MASKED WITH FUNCTION anon.partial(phone,2,$$X-XXX-XX$$,2)'; +``` {.sql parse_query="False"} +SECURITY LABEL FOR anon ON COLUMN customer.lastname + IS 'MASKED WITH FUNCTION anon.dummy_last_name()'; + +SECURITY LABEL FOR anon ON COLUMN customer.phone + IS 'MASKED WITH FUNCTION anon.partial(phone,2,$$X-XXX-XX$$,2)'; ``` ## Apply the rules permanently @@ -98,18 +124,15 @@ SELECT anon.anonymize_table('customer'); | True | ``` sql -SELECT id, - firstname, - lastname, - phone +SELECT id, firstname, lastname, phone FROM customer; ``` | id | firstname | lastname | phone | |-----|-----------|----------|--------------| -| 107 | Sarah | Hessel | 06X-XXX-XX11 | -| 258 | Luke | Hammes | None | -| 341 | Don | Carroll | 34X-XXX-XX23 | +| 107 | Sarah | Abshire | 06X-XXX-XX11 | +| 258 | Luke | Goldner | None | +| 341 | Don | Sauer | 34X-XXX-XX23 | ------------------------------------------------------------------------ @@ -128,7 +151,7 @@ again. Paul realizes that the postcode gives a clear indication of where his customers live. However he would like to have statistics based on their -`postcode area`. +postcode area. **Add a new masking rule to replace the last 3 digits by 'x'.** @@ -144,9 +167,14 @@ date of the customers. Replace all the birth dates by January 1rst, while keeping the real year. -!!! hint +💡 You can use the +[make_date](https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-TABLE) +or +[date_trunc](https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-TABLE) +functions ! - You can use the [make_date] or [date_trunc] functions ! +See + ### E105 - Singling out a customer @@ -156,13 +184,13 @@ instance, we can identify the best client of Paul's boutique with a query like this: ``` sql -WITH best_client AS - (SELECT SUM(amount), - fk_customer_id - FROM payout - GROUP BY fk_customer_id - ORDER BY 1 DESC - LIMIT 1) +WITH best_client AS ( + SELECT SUM(amount), fk_customer_id + FROM payout + GROUP BY fk_customer_id + ORDER BY 1 DESC + LIMIT 1 +) SELECT c.* FROM customer c JOIN best_client b ON (c.id = b.fk_customer_id) @@ -170,11 +198,10 @@ JOIN best_client b ON (c.id = b.fk_customer_id) | id | firstname | lastname | phone | birth | postcode | |-----|-----------|----------|--------------|------------|----------| -| 341 | Don | Carroll | 34X-XXX-XX23 | 1926-06-01 | 04520 | +| 341 | Don | Sauer | 34X-XXX-XX23 | 1926-06-01 | 04520 | -!!! note - - This is called **[Singling Out] a person.** +💡 This is called **[Singling +Out](https://www.pnas.org/content/117/15/8344) a person.** We need to anonymize even further by removing the link between a person and its company. In the `payout` table, this link is materialized by a @@ -190,50 +217,39 @@ the integrity of the data? Find a function that will shuffle the column `fk_company_id` of the `payout` table -!!! tip - - Check out the [static masking] section of the [documentation]. +💡 Check out the [shuffling](static_masking#shuffling) section of the +[documentation](https://postgresql-anonymizer.readthedocs.io/en/stable/). ## Solutions ### S101 ``` sql -SECURITY LABEL -FOR anon ON COLUMN customer.firstname IS 'MASKED WITH FUNCTION anon.dummy_first_name()'; - +SECURITY LABEL FOR anon ON COLUMN customer.firstname +IS 'MASKED WITH FUNCTION anon.dummy_first_name()'; SELECT anon.anonymize_table('customer'); - -SELECT id, - firstname, - lastname +SELECT id, firstname, lastname FROM customer; ``` ### S102 ``` sql -SECURITY LABEL -FOR anon ON COLUMN customer.postcode IS 'MASKED WITH FUNCTION anon.partial(postcode,2,$$xxx$$,0)'; - +SECURITY LABEL FOR anon ON COLUMN customer.postcode +IS 'MASKED WITH FUNCTION anon.partial(postcode,2,$$xxx$$,0)'; SELECT anon.anonymize_table('customer'); - -SELECT id, - firstname, - lastname, - postcode +SELECT id, firstname, lastname, postcode FROM customer; ``` ### S103 ``` sql -SELECT postcode, - COUNT(id) +SELECT postcode, COUNT(id) FROM customer GROUP BY postcode; ``` @@ -245,11 +261,17 @@ GROUP BY postcode; ### S104 -``` sql -SECURITY LABEL FOR anon ON FUNCTION pg_catalog.date_trunc(text,interval) IS 'TRUSTED'; -SECURITY LABEL FOR anon ON COLUMN customer.birth IS $$ MASKED WITH FUNCTION pg_catalog.date_trunc('year',birth) $$; +``` {.sql parse_query="False"} +SECURITY LABEL FOR anon ON FUNCTION pg_catalog.date_trunc(text,interval) + IS 'TRUSTED'; + +SECURITY LABEL FOR anon ON COLUMN customer.birth + IS $$ MASKED WITH FUNCTION pg_catalog.date_trunc('year',birth) $$; + SELECT anon.anonymize_table('customer'); -SELECT id, firstname, lastname, birth FROM customer; + +SELECT id, firstname, lastname, birth +FROM customer; ``` ### S105 @@ -257,7 +279,7 @@ SELECT id, firstname, lastname, birth FROM customer; Let's mix up the values of the `fk_customer_id`: ``` sql -SELECT anon.shuffle_column('payout', 'fk_customer_id', 'id'); +SELECT anon.shuffle_column('payout','fk_customer_id','id'); ``` | shuffle_column | @@ -267,21 +289,21 @@ SELECT anon.shuffle_column('payout', 'fk_customer_id', 'id'); Now let's try to single out the best client again : ``` sql -WITH best_client AS - (SELECT SUM(amount), - fk_customer_id - FROM payout - GROUP BY fk_customer_id - ORDER BY 1 DESC - LIMIT 1) +WITH best_client AS ( + SELECT SUM(amount), fk_customer_id + FROM payout + GROUP BY fk_customer_id + ORDER BY 1 DESC + LIMIT 1 +) SELECT c.* FROM customer c JOIN best_client b ON (c.id = b.fk_customer_id); ``` -| id | firstname | lastname | phone | birth | postcode | -|-----|-----------|----------|--------------|------------|----------| -| 341 | Orland | Lubowitz | 34X-XXX-XX23 | 1926-01-01 | 04xxx | +| id | firstname | lastname | phone | birth | postcode | +|-----|-----------|----------|-------|------------|----------| +| 258 | Lydia | Toy | None | 1951-01-01 | 90xxx | ------------------------------------------------------------------------ diff --git a/docs/tutorials/2-dynamic_masking.md b/docs/tutorials/2-dynamic_masking.md index ab4ca63..dfa7abb 100644 --- a/docs/tutorials/2-dynamic_masking.md +++ b/docs/tutorials/2-dynamic_masking.md @@ -1,17 +1,22 @@ -# 2- How to use Dynamic Masking +# 2- Dynamic Masking -> With Dynamic Masking, the database owner can hide personal data for -> some users, while other users are still allowed to read and write the -> authentic data. +💡 With Dynamic Masking, the database owner can hide personal data for +some users, while other users are still allowed to read and write the +authentic data. + +## Requirements + +**Please check out the [intro](tutorials/0-intro/) of this tutorial if +you haven't read it yet** ## The Story Paul has 2 employees: -- Jack is operating the new sales application, he needs access to the - real data. He is what the GPDR would call a **\"data processor\"**. -- Pierre is a data analyst who runs statistic queries on the database. - He should not have access to any personal data. +- Jack is operating the new sales application, he needs access to the + real data. He is what the GPDR would call a **\"data processor\"**. +- Pierre is a data analyst who runs statistic queries on the database. + He should not have access to any personal data. ## How it works @@ -21,28 +26,36 @@ Paul has 2 employees: In this section, we will learn: -- How to write simple masking rules -- The advantage and limitations of dynamic masking -- The concept of \"Linkability\" of a person +- How to write simple masking rules +- The advantage and limitations of dynamic masking +- The concept of \"Linkability\" of a person ## The `company` table -``` sql +``` {.sql parse_query="False"} + DROP TABLE IF EXISTS supplier CASCADE; + DROP TABLE IF EXISTS company CASCADE; -CREATE TABLE company ( id SERIAL PRIMARY KEY, name TEXT, vat_id TEXT UNIQUE ); + +CREATE TABLE company ( + id SERIAL PRIMARY KEY, + name TEXT, + vat_id TEXT UNIQUE +); ``` ``` sql INSERT INTO company -VALUES (952,'Shadrach', 'FR62684255667'), - (194,E'Johnny\'s Shoe Store','CHE670945644'), - (346,'Capitol Records','GB663829617823') ; +VALUES +(952,'Shadrach', 'FR62684255667'), +(194,E'Johnny\'s Shoe Store','CHE670945644'), +(346,'Capitol Records','GB663829617823') +; ``` ``` sql -SELECT * -FROM company; +SELECT * FROM company; ``` | id | name | vat_id | @@ -53,19 +66,26 @@ FROM company; ## The `supplier` table -``` sql -CREATE TABLE supplier ( id SERIAL PRIMARY KEY, fk_company_id INT REFERENCES company(id), contact TEXT, phone TEXT, job_title TEXT ); +``` {.sql parse_query="False"} +CREATE TABLE supplier ( + id SERIAL PRIMARY KEY, + fk_company_id INT REFERENCES company(id), + contact TEXT, + phone TEXT, + job_title TEXT +); ``` ``` sql INSERT INTO supplier -VALUES (299,194,'Johnny Ryall','597-500-569','CEO'), - (157,346,'George Clinton', '131-002-530','Sales manager') ; +VALUES +(299,194,'Johnny Ryall','597-500-569','CEO'), +(157,346,'George Clinton', '131-002-530','Sales manager') +; ``` ``` sql -SELECT * -FROM supplier; +SELECT * FROM supplier; ``` | id | fk_company_id | contact | phone | job_title | @@ -77,12 +97,10 @@ FROM supplier; ``` sql ALTER DATABASE boutique -SET session_preload_libraries TO 'anon'; - + SET session_preload_libraries TO 'anon'; CREATE EXTENSION IF NOT EXISTS anon; - SELECT anon.init(); ``` @@ -90,24 +108,23 @@ SELECT anon.init(); ### Activate the masking engine -``` sql -ALTER DATABASE boutique SET anon.transparent_dynamic_masking TO true; +``` {.sql parse_query="False"} +ALTER DATABASE boutique + SET anon.transparent_dynamic_masking TO true; ``` ### Masking a role ``` sql -SECURITY LABEL -FOR anon ON ROLE pierre IS 'MASKED'; +SECURITY LABEL FOR anon ON ROLE pierre IS 'MASKED'; -GRANT pg_read_all_data TO pierre; +GRANT pg_read_all_data to pierre; ``` Now connect as Pierre and try to read the supplier table: -``` sql -SELECT * -FROM supplier; +``` {.sql user="pierre"} +SELECT * FROM supplier; ``` | id | fk_company_id | contact | phone | job_title | @@ -123,17 +140,16 @@ data in each table. Connect as Paul and define a masking rule on the supplier table: ``` sql -SECURITY LABEL -FOR anon ON COLUMN supplier.contact IS 'MASKED WITH VALUE $$CONFIDENTIAL$$'; +SECURITY LABEL FOR anon ON COLUMN supplier.contact + IS 'MASKED WITH VALUE $$CONFIDENTIAL$$'; ``` ------------------------------------------------------------------------ Now connect as Pierre and try to read the supplier table again: -``` sql -SELECT * -FROM supplier; +``` {.sql user="pierre"} +SELECT * FROM supplier; ``` | id | fk_company_id | contact | phone | job_title | @@ -145,9 +161,8 @@ FROM supplier; Now connect as Jack and try to read the real data: -``` sql -SELECT * -FROM supplier; +``` {.sql user="jack"} +SELECT * FROM supplier; ``` | id | fk_company_id | contact | phone | job_title | @@ -159,17 +174,19 @@ FROM supplier; ### E201 - Guess who is the CEO of "Johnny's Shoe Store" -Masking the supplier name is clearly not enough to provide anonymity. +Masking the supplier contact is clearly not enough to provide anonymity. -**Connect as Pierre and write a simple SQL query that would reindentify -some suppliers based on their job and their company.** +**Connect as Pierre and write a simple SQL query that joins the +`supplier` and the `company` tables. See how that could reindentify some +suppliers based on their job and their company.** -Company names and job positions are available in many public datasets. A -simple search on Linkedin or Google, would give you the names of the top -executives of most companies.. +With this request we managed to link a person to a company and we know +it's job title. Since company names and job positions are available in +many public datasets: a simple search on Linkedin or Google would give +us the real names of many of the employees of these companies... -> This is called **Linkability**: the ability to connect multiple -> records concerning the same data subject. +💡 This is called **Linkability**: the ability to connect multiple +records concerning the same data subject. ### E202 - Anonymize the companies @@ -177,41 +194,45 @@ We need to anonymize the `company` table, too. Even if they don't contain personal information, some fields can be used to **infer** the identity of their employees... -**Write 2 masking rules for the company table. The first one will -replace the `name` field with a fake name. The second will replace the -`vat_id` with a random sequence of 10 characters** +**Connect as Paul and write 2 masking rules (security labels) for the +company table.** -!!! tip +- The first one will replace the `name` field with a fake name. +- The second rule will replace the `vat_id` with a random sequence of 10 + characters - Go to the[documentation] and look at the [faking functions] and the - [random functions] ! +💡 Go to +the[documentation](https://postgresql-anonymizer.readthedocs.io/en/stable/) +and look at the [faking functions](masking_functions#faking) and the +[random functions](masking_functions#randomization) ! Connect as Pierre and check that he cannot view the real company info. +Connect as Jack and check that he can view the real values. + ### E203 - Pseudonymize the company name Because of dynamic masking, the fake values will be different every time Pierre tries to read the table. Pierre would like to have always the same fake values for a given -company. **This is called pseudonymization.** +company. -**Write a new masking rule over the `vat_id` field by generating 10 -random characters using the md5() function.** +💡 **This is called pseudonymization.** + +**Connect as Paul and write a new masking rule over the `vat_id` field +by generating a hash of 10 characters using the `anon.digest()` +function.** **Write a new masking rule over the `name` field by using a -[pseudonymizing -function](https://postgresql-anonymizer.readthedocs.io/en/stable/masking_functions#pseudonymization).** +[pseudonymizing function](masking_functions#pseudonymization).** ## Solutions ### S201 -``` sql -SELECT s.id, - s.contact, - s.job_title, - c.name +``` {.sql user="pierre"} +SELECT s.id, s.contact, s.job_title, c.name FROM supplier s JOIN company c ON s.fk_company_id = c.id; ``` @@ -224,68 +245,81 @@ JOIN company c ON s.fk_company_id = c.id; ### S202 ``` sql -SECURITY LABEL -FOR anon ON COLUMN company.name IS 'MASKED WITH FUNCTION anon.dummy_company_name()'; +SECURITY LABEL FOR anon ON COLUMN company.name + IS 'MASKED WITH FUNCTION anon.dummy_company_name()'; -SECURITY LABEL -FOR anon ON COLUMN company.vat_id IS 'MASKED WITH FUNCTION anon.random_string(10)'; +SECURITY LABEL FOR anon ON COLUMN company.vat_id +IS 'MASKED WITH FUNCTION anon.random_string(10)'; ``` Now connect as Pierre and read the table again: -``` sql -SELECT * -FROM company; +``` {.sql user="pierre"} +SELECT * FROM company; ``` -| id | name | vat_id | -|-----|------------------------|------------| -| 952 | Thiel and Hudson LLC | kG6CBmpRHZ | -| 194 | Little and Bernier Inc | CRpk4yez0x | -| 346 | Purdy LLC | 92sMlGfRoV | +| id | name | vat_id | +|-----|---------------------|------------| +| 952 | Bashirian LLC | Yg1GmRm0WW | +| 194 | Towne and Sons | IzzSE2QmEC | +| 346 | Cartwright and Sons | LjTIY7QrBm | Pierre will see different "fake data" every time he reads the table: -``` sql -SELECT * -FROM company; +``` {.sql user="pierre"} +SELECT * FROM company; +``` + +| id | name | vat_id | +|-----|----------------------|------------| +| 952 | Wolf and Haley Group | T0UjIXqLu5 | +| 194 | Rippin Inc | EpB97liUYC | +| 346 | Weber and Bayer LLC | flyM5UaRPV | + +Jack still sees the real data + +``` {.sql user="jack"} +SELECT * FROM company; ``` -| id | name | vat_id | -|-----|----------------------------------|------------| -| 952 | Yundt and Sons | k2HE9JaEpT | -| 194 | Schuster and Konopelski and Sons | 8SPYpX1866 | -| 346 | Ziemann and Wisoky Inc | 1G2Ot6LjUE | +| id | name | vat_id | +|-----|----------------------|----------------| +| 952 | Shadrach | FR62684255667 | +| 194 | Johnny\'s Shoe Store | CHE670945644 | +| 346 | Capitol Records | GB663829617823 | ### S203 ``` sql -SECURITY LABEL -FOR anon ON COLUMN company.name IS 'MASKED WITH FUNCTION anon.pseudo_company(id)'; +SECURITY LABEL FOR anon ON COLUMN company.vat_id +IS $$ MASKED WITH FUNCTION anon.left(anon.digest(vat_id, 'xxx', 'md5'),10) $$; +``` + +``` sql +SECURITY LABEL FOR anon ON COLUMN company.name + IS 'MASKED WITH FUNCTION anon.pseudo_company(id)'; ``` Connect as Pierre and read the table multiple times: -``` sql -SELECT * -FROM company; +``` {.sql user="pierre"} +SELECT * FROM company; ``` | id | name | vat_id | |-----|-----------------|------------| -| 952 | Wilkinson LLC | Ifsi290QIn | -| 194 | Johnson PLC | LzgMedlx2A | -| 346 | Young-Carpenter | HbcZDZ2hTT | +| 952 | Wilkinson LLC | 2db762afa4 | +| 194 | Johnson PLC | 61fddf8d83 | +| 346 | Young-Carpenter | 86fe3f164c | -``` sql -SELECT * -FROM company; +``` {.sql user="pierre"} +SELECT * FROM company; ``` | id | name | vat_id | |-----|-----------------|------------| -| 952 | Wilkinson LLC | YQPTtQXfWM | -| 194 | Johnson PLC | EVwSatDJ51 | -| 346 | Young-Carpenter | UzRp4fKpSO | +| 952 | Wilkinson LLC | 2db762afa4 | +| 194 | Johnson PLC | 61fddf8d83 | +| 346 | Young-Carpenter | 86fe3f164c | Now the fake company name is always the same. diff --git a/docs/tutorials/3-anonymous_dumps.md b/docs/tutorials/3-anonymous_dumps.md index d9c9cdd..447b5ae 100644 --- a/docs/tutorials/3-anonymous_dumps.md +++ b/docs/tutorials/3-anonymous_dumps.md @@ -1,8 +1,8 @@ # 3- Anonymous Dumps -> In many situation, what we want is basically to export the anonymized -> data into another database (for testing or to produce statistics). We -> will simply use pg_dump for that ! +💡 In many situation, what we want is basically to export the anonymized +data into another database (for testing or to produce statistics). We +will simply use pg_dump for that ! ## The Story @@ -20,33 +20,47 @@ information contained in the comment section. ## Learning Objective -- Extract the anonymized data from the database -- Write a custom masking function to handle a JSON field. +- Extract the anonymized data from the database +- Write a custom masking function to handle a JSON field. ## Load the data ``` sql DROP TABLE IF EXISTS website_comment CASCADE; - -CREATE TABLE website_comment (id SERIAL PRIMARY KEY, - message JSONB); +CREATE TABLE website_comment ( + id SERIAL PRIMARY KEY, + message JSONB +); ``` ``` sql INSERT INTO website_comment -VALUES (1, json_build_object('meta', json_build_object('name', 'Lee Perry', 'ip_addr','40.87.29.113'), 'content', 'Hello Nasty!')), - (2, json_build_object('meta', json_build_object('name', '', 'email', 'biz@bizmarkie.com'), 'content', 'Great Shop')), - (3,json_build_object('meta', json_build_object('name','Jimmy'), 'content','Hi ! This is me, Jimmy James')); +VALUES + (1, json_build_object( + 'meta', json_build_object( + 'name', 'Lee Perry', + 'ip_addr','40.87.29.113'), + 'content', 'Hello Nasty!')), + (2, json_build_object( + 'meta', json_build_object( + 'name', '', + 'email', 'biz@bizmarkie.com'), + 'content', 'Great Shop')), + (3,json_build_object( + 'meta', json_build_object( + 'name','Jimmy'), + 'content','Hi ! This is me, Jimmy James')); ``` Check the content of the website comments: ``` sql -SELECT message->'meta'->'name' AS name, - message->'content' AS content +SELECT + message->'meta'->'name' AS name, + message->'content' AS content FROM website_comment -ORDER BY id ASC +ORDER BY id ASC; ``` | name | content | @@ -74,15 +88,16 @@ there's no way to extract personal data properly. ------------------------------------------------------------------------ -We can *clean* the comment column simply by removing the `content` key! +We can *clean* the comment column simply by removing the `content` key +in the `message` column ! ``` sql -SELECT message - ARRAY['content'] +SELECT message - ARRAY['content'] AS message_without_content FROM website_comment WHERE id=1; ``` -| ?column? | +| message_without_content | |----------------------------------------------------------------------| | {\'meta\': {\'name\': \'Lee Perry\', \'ip_addr\': \'40.87.29.113\'}} | @@ -96,8 +111,7 @@ add functions in this schema. ``` sql CREATE SCHEMA IF NOT EXISTS my_masks; -SECURITY LABEL -FOR anon ON SCHEMA my_masks IS 'TRUSTED'; +SECURITY LABEL FOR anon ON SCHEMA my_masks IS 'TRUSTED'; ``` ------------------------------------------------------------------------ @@ -105,7 +119,13 @@ FOR anon ON SCHEMA my_masks IS 'TRUSTED'; Now we can write a function that remove the message content: ``` sql -CREATE OR REPLACE FUNCTION my_masks.remove_content(j JSONB) RETURNS JSONB AS $func$ SELECT j - ARRAY['content'] $func$ LANGUAGE SQL ; +CREATE OR REPLACE FUNCTION my_masks.remove_content(j JSONB) +RETURNS JSONB +AS $func$ + SELECT j - ARRAY['content'] +$func$ +LANGUAGE SQL +; ``` ------------------------------------------------------------------------ @@ -114,7 +134,7 @@ Let's try it! ``` sql SELECT my_masks.remove_content(message) -FROM website_comment +FROM website_comment; ``` | remove_content | @@ -126,8 +146,8 @@ FROM website_comment And now we can use it in a masking rule: ``` sql -SECURITY LABEL -FOR anon ON COLUMN website_comment.message IS 'MASKED WITH FUNCTION my_masks.remove_content(message)'; +SECURITY LABEL FOR anon ON COLUMN website_comment.message +IS 'MASKED WITH FUNCTION my_masks.remove_content(message)'; ``` Then we need to create a dedicated role to export the masked data. We @@ -137,12 +157,9 @@ that this role is masked. ``` sql CREATE ROLE anon_dumper LOGIN PASSWORD 'CHANGEME'; +ALTER ROLE anon_dumper SET anon.transparent_dynamic_masking TO TRUE; -ALTER ROLE anon_dumper -SET anon.transparent_dynamic_masking TO TRUE; - -SECURITY LABEL -FOR anon ON ROLE anon_dumper IS 'MASKED'; +SECURITY LABEL FOR anon ON ROLE anon_dumper IS 'MASKED'; GRANT pg_read_all_data TO anon_dumper; ``` @@ -170,23 +187,27 @@ pg_dump -U anon_dumper boutique --table=website_comment > /tmp/dump.sql Create a database named `boutique_anon` and transfer the entire database into it. -### E302 - Pseudonymize the meta fields of the comments +### E302 - Remove the email address + +Replace the `remove_content` function with a better one called +`remove_content_and_ip` that will nullify the `email` key. + +💡 HINT: you can use `jsonb_set(message, '{meta, email}', '{}')` to +remove the email value. + +### E303 - Pseudonymize the IP address Pierre plans to extract general information from the metadata. For instance, he wants to calculate the number of unique visitors based on -the different IP addresses. But an IP address is an **indirect -identifier**, so Paul needs to anonymize this field while maintaining -the fact that some values appear multiple times. - -Replace the `remove_content` function with a better one called -`clean_comment` that will: +the different IP addresses. -- Remove the content key -- Replace the `name` value with a fake last name -- Replace the `ip_address` value with its MD5 signature -- Nullify the `email` key +But an IP address is an **indirect identifier**, so Paul needs to +anonymize this field while maintaining the fact that some values appear +multiple times. -> HINT: Look at the `jsonb_set()` and `jsonb_build_object()` functions +💡 HINT: First you can create a new `meta` object using +`jsonb_build_object()` and then use function `jsonb_set` replace the +`meta` key ## Solutions @@ -208,7 +229,52 @@ psql -U paul boutique_anon -c 'SELECT COUNT(*) FROM company' ### S302 ``` sql -CREATE OR REPLACE FUNCTION my_masks.clean_comment(message JSONB) RETURNS JSONB VOLATILE LANGUAGE SQL AS $func$ SELECT jsonb_set( message, ARRAY['meta'], jsonb_build_object( 'name',anon.fake_last_name(), 'ip_address', md5((message->'meta'->'ip_addr')::TEXT), 'email', NULL ) ) - ARRAY['content']; $func$; +CREATE OR REPLACE FUNCTION my_masks.remove_content_and_ip(message JSONB) +RETURNS JSONB +VOLATILE +LANGUAGE SQL +AS $func$ +SELECT + jsonb_set(message, '{meta, email}', '{}') + - ARRAY['content']; +$func$; +``` + +``` sql +SELECT my_masks.remove_content_and_ip(message) +FROM website_comment; +``` + +| remove_content_and_ip | +|----| +| {\'meta\': {\'name\': \'Lee Perry\', \'email\': {}, \'ip_addr\': \'40.87.29.113\'}} | +| {\'meta\': {\'name\': \'\', \'email\': {}}} | +| {\'meta\': {\'name\': \'Jimmy\', \'email\': {}}} | + +``` sql +SECURITY LABEL FOR anon ON COLUMN website_comment.message +IS 'MASKED WITH FUNCTION my_masks.remove_content_and_ip(message)'; +``` + +### S303 + +``` sql +CREATE OR REPLACE FUNCTION my_masks.clean_comment(message JSONB) +RETURNS JSONB +VOLATILE +LANGUAGE SQL +AS $func$ +SELECT + jsonb_set( + message, + ARRAY['meta'], + jsonb_build_object( + 'name',anon.fake_last_name(), + 'ip_address', md5((message->'meta'->'ip_addr')::TEXT), + 'email', NULL + ) + ) - ARRAY['content']; +$func$; ``` ``` sql @@ -218,11 +284,11 @@ FROM website_comment; | clean_comment | |----| -| {\'meta\': {\'name\': \'Hicks\', \'email\': None, \'ip_address\': \'1d8cbcdef988d55982af1536922ddcd1\'}} | -| {\'meta\': {\'name\': \'Galloway\', \'email\': None, \'ip_address\': None}} | -| {\'meta\': {\'name\': \'Grant\', \'email\': None, \'ip_address\': None}} | +| {\'meta\': {\'name\': \'Gill\', \'email\': None, \'ip_address\': \'1d8cbcdef988d55982af1536922ddcd1\'}} | +| {\'meta\': {\'name\': \'Henson\', \'email\': None, \'ip_address\': None}} | +| {\'meta\': {\'name\': \'Mcmahon\', \'email\': None, \'ip_address\': None}} | ``` sql -SECURITY LABEL -FOR anon ON COLUMN website_comment.message IS 'MASKED WITH FUNCTION my_masks.clean_comment(message)'; +SECURITY LABEL FOR anon ON COLUMN website_comment.message +IS 'MASKED WITH FUNCTION my_masks.clean_comment(message)'; ``` diff --git a/docs/tutorials/4-generalization.md b/docs/tutorials/4-generalization.md index 4929f1d..51ce80b 100644 --- a/docs/tutorials/4-generalization.md +++ b/docs/tutorials/4-generalization.md @@ -1,10 +1,9 @@ -# 4 - Generalization +# 4- Generalization -> The main idea of generalization is to `blur` the original data. For -> example, instead of saying `Mister X was born on July 25, 1989`, we -> can say `Mister X was born is the 80's`. The information is still -> true, but it is less precise and it can\'t be used to reidentify the -> subject. +💡 The main idea of generalization is to `blur` the original data. For +example, instead of saying `Mister X was born on July 25, 1989`, we can +say `Mister X was born is the 80's`. The information is still true, but +it is less precise and it can't be used to reidentify the subject. ## The Story @@ -22,30 +21,42 @@ generalized views to Pierre. In this section, we will learn: -- The difference between masking and generalization -- The concept of `K-anonymity` +- The difference between masking and generalization +- The concept of `K-anonymity` ## The `employee` table -``` sql +``` {.sql parse_query="False"} DROP TABLE IF EXISTS employee CASCADE; -CREATE TABLE employee ( id INT PRIMARY KEY, full_name TEXT, first_day DATE, last_day DATE, height INT, hair TEXT, eyes TEXT, size TEXT, asthma BOOLEAN, CHECK(hair = ANY(ARRAY['bald','blond','dark','red'])), CHECK(eyes = ANY(ARRAY['blue','green','brown'])) , CHECK(size = ANY(ARRAY['S','M','L','XL','XXL'])) ); + +CREATE TABLE employee ( + id INT PRIMARY KEY, + full_name TEXT, + first_day DATE, last_day DATE, + height INT, + hair TEXT, eyes TEXT, size TEXT, + asthma BOOLEAN, + CHECK(hair = ANY(ARRAY['bald','blond','dark','red'])), + CHECK(eyes = ANY(ARRAY['blue','green','brown'])) , + CHECK(size = ANY(ARRAY['S','M','L','XL','XXL'])) +); ``` -!!! danger This is awkward and illegal. +🚨 This is awkward and illegal. Loading the data: ``` sql INSERT INTO employee -VALUES (1,'Luna Dickens','2018-07-22','2018-12-15',180,'blond','blue','L',TRUE), - (2,'Paul Wolf','2020-01-15',NULL,177,'bald','brown','M',FALSE), - (3,'Rowan Hoeger','2018-12-01','2018-12-15',202,'dark','blue','XXL',TRUE) ; + VALUES +(1,'Luna Dickens','2018-07-22','2018-12-15',180,'blond','blue','L',True), +(2,'Paul Wolf','2020-01-15',NULL,177,'bald','brown','M',False), +(3,'Rowan Hoeger','2018-12-01','2018-12-15',202,'dark','blue','XXL',True) +; ``` ``` sql -SELECT count(*) -FROM employee; +SELECT count(*) FROM employee; ``` | count | @@ -53,11 +64,7 @@ FROM employee; | 3 | ``` sql -SELECT full_name, - first_day, - hair, - SIZE, - asthma +SELECT full_name,first_day, hair, size, asthma FROM employee LIMIT 3; ``` @@ -78,10 +85,8 @@ He provides the following view to Pierre. ``` sql DROP MATERIALIZED VIEW IF EXISTS v_asthma_eyes; - CREATE MATERIALIZED VIEW v_asthma_eyes AS -SELECT eyes, - asthma +SELECT eyes, asthma FROM employee; ``` @@ -100,9 +105,9 @@ LIMIT 3; Pierre can now write queries over this view. ``` sql -SELECT eyes, - 100*COUNT(1) FILTER ( - WHERE asthma) / COUNT(1) AS asthma_rate +SELECT + eyes, + 100*COUNT(1) FILTER (WHERE asthma) / COUNT(1) AS asthma_rate FROM v_asthma_eyes GROUP BY eyes; ``` @@ -112,18 +117,28 @@ GROUP BY eyes; | brown | 0 | | blue | 100 | -Pierre just proved that asthma is caused by green eyes. +Pierre just proved that asthma is caused by blue eyes ;-) ## K-Anonymity The `asthma` and `eyes` columns are considered as indirect identifiers. +Indirect personal identifiers (or "quasi-identifiers") are pieces of +information that, when combined with other data can identify an +individual. Examples of indirect identifiers include: Date of birth, +Gender, Zip code, etc. + +With PostgreSQL Anonymizer, we can declare that a column is an indirect +identifiers, like this: + ``` sql -SECURITY LABEL -FOR k_anonymity ON COLUMN v_asthma_eyes.eyes IS 'INDIRECT IDENTIFIER'; +SECURITY LABEL FOR k_anonymity + ON COLUMN v_asthma_eyes.eyes + IS 'INDIRECT IDENTIFIER'; -SECURITY LABEL -FOR k_anonymity ON COLUMN v_asthma_eyes.asthma IS 'INDIRECT IDENTIFIER'; +SECURITY LABEL FOR k_anonymity + ON COLUMN v_asthma_eyes.asthma + IS 'INDIRECT IDENTIFIER'; ``` ``` sql @@ -134,8 +149,8 @@ SELECT anon.k_anonymity('v_asthma_eyes'); |-------------| | 1 | -The v_asthma_eyes has \'2-anonymity\'. This means that each -quasi-identifier combination (the \'eyes-asthma\' tuples) occurs in at +The v_asthma_eyes has '2-anonymity'. This means that each +quasi-identifier combination (the 'eyes-asthma' tuples) occurs in at least 2 records for a dataset. In other words, it means that each individual in the view cannot be @@ -143,12 +158,16 @@ distinguished from at least 1 (k-1) other individual. ## Range and Generalization functions +Now let's add another view over the `employee` table. + +We will generalize the dates of to keep only the month and year. + ``` sql DROP MATERIALIZED VIEW IF EXISTS v_staff_per_month; - CREATE MATERIALIZED VIEW v_staff_per_month AS -SELECT anon.generalize_daterange(first_day, 'month') AS first_day, - anon.generalize_daterange(last_day, 'month') AS last_day +SELECT + anon.generalize_daterange(first_day,'month') AS first_day, + anon.generalize_daterange(last_day,'month') AS last_day FROM employee; ``` @@ -168,8 +187,12 @@ Pierre can write a query to find how many employees were hired in november 2021. ``` sql -SELECT COUNT(1) FILTER ( - WHERE make_date(2019, 11, 1) BETWEEN lower(first_day) AND COALESCE(upper(last_day), now()) ) +SELECT COUNT(1) + FILTER ( + WHERE make_date(2019,11,1) + BETWEEN lower(first_day) + AND COALESCE(upper(last_day),now()) + ) FROM v_staff_per_month; ``` @@ -180,21 +203,23 @@ FROM v_staff_per_month; ### Declaring the indirect identifiers Now let's check the k-anonymity of this view by declaring which columns -are indirect identifiers. +are indirect identifiers : ``` sql -SECURITY LABEL -FOR k_anonymity ON COLUMN v_staff_per_month.first_day IS 'INDIRECT IDENTIFIER'; - -SECURITY LABEL -FOR k_anonymity ON COLUMN v_staff_per_month.last_day IS 'INDIRECT IDENTIFIER'; +SECURITY LABEL FOR k_anonymity + ON COLUMN v_staff_per_month.first_day + IS 'INDIRECT IDENTIFIER'; +SECURITY LABEL FOR k_anonymity + ON COLUMN v_staff_per_month.last_day + IS 'INDIRECT IDENTIFIER'; SELECT anon.k_anonymity('v_staff_per_month'); ``` -In this case, the k factor is 1 which means that at least one unique -individual can be identified directly by his/her first and last dates. +In this case, the k factor is 1 which means that there is at least one +unique individual who be identified directly by his/her first and last +dates. ## Exercises @@ -221,15 +246,17 @@ What is the k-anonymity of `v_staff_per_month_years`? ``` sql DROP MATERIALIZED VIEW IF EXISTS v_staff_per_year; - CREATE MATERIALIZED VIEW v_staff_per_year AS -SELECT int4range(extract(YEAR - FROM first_day)::INT, extract(YEAR - FROM last_day)::INT, '[]') AS period +SELECT + int4range( + extract(year from first_day)::INT, + extract(year from last_day)::INT, + '[]' + ) AS period FROM employee; ``` -!!! tip '\[\]' will include the upper bound +💡 `'[]'` will include the upper bound ------------------------------------------------------------------------ @@ -248,13 +275,16 @@ LIMIT 3; ### S402 ``` sql -SELECT YEAR, - COUNT(1) FILTER ( - WHERE YEAR <@ period ) -FROM generate_series(2018, 2021) YEAR, - v_staff_per_year -GROUP BY YEAR -ORDER BY YEAR ASC; +SELECT + year, + COUNT(1) FILTER ( + WHERE year <@ period + ) +FROM + generate_series(2018,2021) year, + v_staff_per_year +GROUP BY year +ORDER BY year ASC; ``` | year | count | @@ -267,9 +297,9 @@ ORDER BY YEAR ASC; ### S403 ``` sql -SECURITY LABEL -FOR k_anonymity ON COLUMN v_staff_per_year.period IS 'INDIRECT IDENTIFIER'; - +SECURITY LABEL FOR k_anonymity + ON COLUMN v_staff_per_year.period + IS 'INDIRECT IDENTIFIER'; SELECT anon.k_anonymity('v_staff_per_year'); ``` diff --git a/docs/tutorials/9-conclusion.md b/docs/tutorials/9-conclusion.md index 68fb1e1..8745fa3 100644 --- a/docs/tutorials/9-conclusion.md +++ b/docs/tutorials/9-conclusion.md @@ -4,11 +4,12 @@ ## Clean up ! -``` sql +``` {.sql user="postgres" dbname="postgres"} DROP DATABASE IF EXISTS boutique; ``` -``` sql +``` {.sql user="postgres" dbname="postgres"} + REASSIGN OWNED BY jack TO postgres; REASSIGN OWNED BY paul TO postgres; @@ -16,13 +17,10 @@ REASSIGN OWNED BY paul TO postgres; REASSIGN OWNED BY pierre TO postgres; ``` -``` sql +``` {.sql user="postgres" dbname="postgres"} DROP ROLE IF EXISTS jack; - DROP ROLE IF EXISTS paul; - DROP ROLE IF EXISTS pierre; - DROP ROLE IF EXISTS dump_anon; ``` @@ -30,8 +28,8 @@ DROP ROLE IF EXISTS dump_anon; Other projects you may like -- [pg_sample](https://github.com/mla/pg_sample) : extract a small - dataset from a larger PostgreSQL database +- [pg_sample](https://github.com/mla/pg_sample) : extract a small + dataset from a larger PostgreSQL database ## Help Wanted! diff --git a/docs/tutorials/__DO_NOT_MODIFY_THESE_FILES__.md b/docs/tutorials/__DO_NOT_MODIFY_THESE_FILES__.md new file mode 100644 index 0000000..fb39b2d --- /dev/null +++ b/docs/tutorials/__DO_NOT_MODIFY_THESE_FILES__.md @@ -0,0 +1,10 @@ +# DO NOT MODIFY THESE FILES + +The files in the `docs/tutorial` folder are artifacts generated based +on the source files in `docs/runbooks`. + +If you want to improve the tutorial, edit the `docs/runbooks/*.md` files. + +And then run `make tutorial` to update the artifacts. + + diff --git a/mkdocs.yml b/mkdocs.yml index a8a1d87..4f2cbcb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -16,6 +16,7 @@ nav: - Masking Methods: - 'Static Masking': 'static_masking.md' - 'Dynamic Masking': 'dynamic_masking.md' + - 'Replica Masking': 'replica_masking.md' - 'Anonymous Dumps': 'anonymous_dumps.md' - 'Masking Views': 'masking_views.md' - 'Masking Data Wrappers': 'masking_data_wrappers.md' diff --git a/patches/anon.patch b/patches/anon.patch new file mode 100644 index 0000000..60eeda9 --- /dev/null +++ b/patches/anon.patch @@ -0,0 +1,646 @@ +diff --git a/sql/anon.sql b/sql/anon.sql +index 0cdc769..85a58a6 100644 +--- a/sql/anon.sql ++++ b/sql/anon.sql +@@ -1141,3 +1141,9 @@ $$ + -- TODO : https://en.wikipedia.org/wiki/L-diversity + + -- TODO : https://en.wikipedia.org/wiki/T-closeness ++ ++-- NEON Patches ++ ++GRANT ALL ON SCHEMA anon to neon_superuser; ++GRANT ALL ON ALL TABLES IN SCHEMA anon TO neon_superuser; ++-- GRANT SET ON PARAMETER anon.transparent_dynamic_masking TO neon_superuser; +diff --git a/sql/init.sql b/sql/init.sql +index 7da6553..7961984 100644 +--- a/sql/init.sql ++++ b/sql/init.sql +@@ -74,50 +74,49 @@ $$ + + SECURITY LABEL FOR anon ON FUNCTION anon.load_csv IS 'UNTRUSTED'; + +--- load fake data from a given path +-CREATE OR REPLACE FUNCTION anon.init( +- datapath TEXT +-) ++CREATE OR REPLACE FUNCTION anon.load_fake_data() + RETURNS BOOLEAN + AS $$ + DECLARE +- datapath_check TEXT; + success BOOLEAN; ++ sharedir TEXT; ++ datapath TEXT; + BEGIN + +- IF anon.is_initialized() THEN +- RAISE NOTICE 'The anon extension is already initialized.'; +- RETURN TRUE; +- END IF; ++ datapath := '/extension/anon/'; ++ -- find the local extension directory ++ SELECT setting INTO sharedir ++ FROM pg_catalog.pg_config ++ WHERE name = 'SHAREDIR'; + + SELECT bool_or(results) INTO success + FROM unnest(array[ +- anon.load_csv('anon.identifiers_category',datapath||'/identifiers_category.csv'), +- anon.load_csv('anon.identifier',datapath ||'/identifier.csv'), +- anon.load_csv('anon.address',datapath ||'/address.csv'), +- anon.load_csv('anon.city',datapath ||'/city.csv'), +- anon.load_csv('anon.company',datapath ||'/company.csv'), +- anon.load_csv('anon.country',datapath ||'/country.csv'), +- anon.load_csv('anon.email', datapath ||'/email.csv'), +- anon.load_csv('anon.first_name',datapath ||'/first_name.csv'), +- anon.load_csv('anon.iban',datapath ||'/iban.csv'), +- anon.load_csv('anon.last_name',datapath ||'/last_name.csv'), +- anon.load_csv('anon.postcode',datapath ||'/postcode.csv'), +- anon.load_csv('anon.siret',datapath ||'/siret.csv'), +- anon.load_csv('anon.lorem_ipsum',datapath ||'/lorem_ipsum.csv') ++ anon.load_csv('anon.identifiers_category',sharedir || datapath || '/identifiers_category.csv'), ++ anon.load_csv('anon.identifier',sharedir || datapath || '/identifier.csv'), ++ anon.load_csv('anon.address',sharedir || datapath || '/address.csv'), ++ anon.load_csv('anon.city',sharedir || datapath || '/city.csv'), ++ anon.load_csv('anon.company',sharedir || datapath || '/company.csv'), ++ anon.load_csv('anon.country',sharedir || datapath || '/country.csv'), ++ anon.load_csv('anon.email', sharedir || datapath || '/email.csv'), ++ anon.load_csv('anon.first_name',sharedir || datapath || '/first_name.csv'), ++ anon.load_csv('anon.iban',sharedir || datapath || '/iban.csv'), ++ anon.load_csv('anon.last_name',sharedir || datapath || '/last_name.csv'), ++ anon.load_csv('anon.postcode',sharedir || datapath || '/postcode.csv'), ++ anon.load_csv('anon.siret',sharedir || datapath || '/siret.csv'), ++ anon.load_csv('anon.lorem_ipsum',sharedir || datapath || '/lorem_ipsum.csv') + ]) results; + RETURN success; +- + END; + $$ +- LANGUAGE PLPGSQL ++ LANGUAGE plpgsql + VOLATILE + RETURNS NULL ON NULL INPUT +- PARALLEL UNSAFE -- because load_csv is unsafe +- SECURITY INVOKER ++ PARALLEL UNSAFE -- because of the EXCEPTION ++ SECURITY DEFINER + SET search_path='' + ; +-SECURITY LABEL FOR anon ON FUNCTION anon.init(TEXT) IS 'UNTRUSTED'; ++ ++SECURITY LABEL FOR anon ON FUNCTION anon.load_fake_data IS 'UNTRUSTED'; + + -- People tend to forget the anon.init() step + -- This is a friendly notice for them +@@ -144,7 +143,7 @@ SECURITY LABEL FOR anon ON FUNCTION anon.notice_if_not_init IS 'UNTRUSTED'; + CREATE OR REPLACE FUNCTION anon.load(TEXT) + RETURNS BOOLEAN AS + $$ +- SELECT anon.init($1); ++ SELECT anon.init(); + $$ + LANGUAGE SQL + VOLATILE +@@ -159,16 +158,16 @@ SECURITY LABEL FOR anon ON FUNCTION anon.load(TEXT) IS 'UNTRUSTED'; + CREATE OR REPLACE FUNCTION anon.init() + RETURNS BOOLEAN + AS $$ +- WITH conf AS ( +- -- find the local extension directory +- SELECT setting AS sharedir +- FROM pg_catalog.pg_config +- WHERE name = 'SHAREDIR' +- ) +- SELECT anon.init(conf.sharedir || '/extension/anon/') +- FROM conf; ++BEGIN ++ IF anon.is_initialized() THEN ++ RAISE NOTICE 'The anon extension is already initialized.'; ++ RETURN TRUE; ++ END IF; ++ ++ RETURN anon.load_fake_data(); ++END; + $$ +- LANGUAGE SQL ++ LANGUAGE plpgsql + VOLATILE + PARALLEL UNSAFE -- because init is unsafe + SECURITY INVOKER +@@ -264,3 +263,22 @@ $$ + ; + + SECURITY LABEL FOR anon ON FUNCTION anon.unload IS 'UNTRUSTED'; ++ ++ ++CREATE OR REPLACE FUNCTION anon.toggle_transparent_dynamic_masking( ++ dbname TEXT, ++ toggle BOOLEAN DEFAULT TRUE ++) ++RETURNS VOID AS ++$$ ++BEGIN ++ EXECUTE format('ALTER DATABASE %I SET anon.transparent_dynamic_masking TO %s', dbname, toggle::TEXT); ++END; ++$$ ++ LANGUAGE plpgsql ++ VOLATILE ++ SECURITY DEFINER ++ SET search_path='' ++; ++ ++SECURITY LABEL FOR anon ON FUNCTION anon.toggle_transparent_dynamic_masking IS 'UNTRUSTED'; +diff --git a/src/dummy.rs b/src/dummy.rs +index a514bb4..e448031 100644 +--- a/src/dummy.rs ++++ b/src/dummy.rs +@@ -106,11 +106,8 @@ macro_rules! declare_l10n_fn_String { + + #[pg_extern] + pub fn $name() -> String { +- let locale = $crate::guc::ANON_DUMMY_LOCALE +- .get() +- .unwrap() +- .to_str() +- .expect("Should be a string"); ++ let guc_value = $crate::guc::ANON_DUMMY_LOCALE.get().unwrap(); ++ let locale = guc_value.to_str().expect("Should be a string"); + dummy!($struct, locale) + } + }; +@@ -132,11 +129,8 @@ macro_rules! declare_l10n_fn_with_range_to_string { + + #[pg_extern] + pub fn $name(r: pgrx::Range) -> String { +- let locale = $crate::guc::ANON_DUMMY_LOCALE +- .get() +- .unwrap() +- .to_str() +- .expect("Should be a string"); ++ let guc_value = $crate::guc::ANON_DUMMY_LOCALE.get().unwrap(); ++ let locale = guc_value.to_str().expect("Should be a string"); + return $crate::dummy_with_range!($struct, locale, r); + } + }; +diff --git a/src/guc.rs b/src/guc.rs +index 74d3822..6c04383 100644 +--- a/src/guc.rs ++++ b/src/guc.rs +@@ -3,22 +3,16 @@ + //---------------------------------------------------------------------------- + + use pgrx::*; +-use std::ffi::CStr; ++use std::ffi::{c_void, CStr, CString}; + +-pub static ANON_DUMMY_LOCALE: GucSetting> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"en_US\0") +- })); ++pub static ANON_DUMMY_LOCALE: GucSetting> = ++ GucSetting::>::new(Some(c"en_US")); + +-pub static ANON_K_ANONYMITY_PROVIDER: GucSetting> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"k_anonymity\0") +- })); ++pub static ANON_K_ANONYMITY_PROVIDER: GucSetting> = ++ GucSetting::>::new(Some(c"k_anonymity")); + +-pub static ANON_MASKING_POLICIES: GucSetting> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"\0") +- })); ++pub static ANON_MASKING_POLICIES: GucSetting> = ++ GucSetting::>::new(Some(c"")); + + pub static ANON_PRIVACY_BY_DEFAULT: GucSetting = GucSetting::::new(false); + +@@ -31,135 +25,234 @@ pub static ANON_TRANSPARENT_DYNAMIC_MASKING: GucSetting = GucSetting::> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"sha256\0") +- })); ++static ANON_ALGORITHM: GucSetting> = ++ GucSetting::>::new(Some(c"sha256")); + +-static ANON_SALT: GucSetting> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"\0") +- })); ++static ANON_SALT: GucSetting> = GucSetting::>::new(Some(c"")); + +-static ANON_SOURCE_SCHEMA: GucSetting> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"public\0") +- })); ++static ANON_SOURCE_SCHEMA: GucSetting> = ++ GucSetting::>::new(Some(c"public")); + +-static ANON_MASK_SCHEMA: GucSetting> = +- GucSetting::>::new(Some(unsafe { +- CStr::from_bytes_with_nul_unchecked(b"mask\0") +- })); ++static ANON_MASK_SCHEMA: GucSetting> = ++ GucSetting::>::new(Some(c"mask")); ++ ++unsafe extern "C-unwind" fn check_bool_guc_hook( ++ _newval: *mut bool, ++ _extra: *mut *mut c_void, ++ source: u32, ++) -> bool { ++ unsafe { ++ // The sources that we allow are: ++ // 1. PGC_S_DEFAULT (0) -> for default boot up source, likely new session or server. ++ // 2. PGC_S_DATABASE (6) -> a GUC set for a particular database ++ // 3. PGC_S_USER (7) -> a GUC set for a particular role ++ // 4. PGC_S_DATABASE_USER (8) -> a GUC set for a particular role in a particular database ++ // This check only allows sources that load a variable, not ones that try to alter it. ++ // Sources that try to alter it are: ++ // 1. PGC_S_FILE (3) -> ALTER SYSTEM ++ // 2. PGC_S_TEST (12) -> ALTER ROLE/DATABASE ++ // 3. PGC_S_SESSION (13) -> SET ... ++ // TODO (thesuhas): Does PGC_S_GLOBAL need to be added to whitelisted sources? ++ pg_sys::info!("Source: {}", source); ++ if source == 0 || source == 6 || source == 7 || source == 8 { ++ return true; ++ } ++ let oid = pg_sys::GetUserId(); ++ let user_name = CStr::from_ptr(pg_sys::GetUserNameFromId(oid, true)); ++ let user_str = user_name.to_str().unwrap(); ++ pg_sys::info!("user: {} trying to change boolean guc", user_str); ++ if pg_sys::superuser() || user_str == "neon_superuser" || user_str == "neondb_owner" { ++ return true; ++ } ++ pg_sys::ereport!( ++ PgLogLevel::ERROR, ++ PgSqlErrorCode::ERRCODE_INSUFFICIENT_PRIVILEGE, ++ "You are not authorized to change this GUC" ++ ); ++ false ++ } ++} ++ ++unsafe extern "C-unwind" fn check_string_guc_hook( ++ _newval: *mut *mut libc::c_char, ++ _extra: *mut *mut c_void, ++ source: u32, ++) -> bool { ++ unsafe { ++ // The sources that we allow are: ++ // 1. PGC_S_DEFAULT (0) -> for default boot up source, likely new session or server. ++ // 2. PGC_S_DATABASE (6) -> a GUC set for a particular database ++ // 3. PGC_S_USER (7) -> a GUC set for a particular role ++ // 4. PGC_S_DATABASE_USER (8) -> a GUC set for a particular role in a particular database ++ // This check only allows sources that load a variable, not ones that try to alter it. ++ // Sources that try to alter it are: ++ // 1. PGC_S_FILE (3) -> ALTER SYSTEM ++ // 2. PGC_S_TEST (12) -> ALTER ROLE/DATABASE ++ // 3. PGC_S_SESSION (13) -> SET ... ++ pg_sys::info!("Source: {}", source); ++ if source == 0 || source == 6 || source == 7 || source == 8 { ++ return true; ++ } ++ let oid = pg_sys::GetUserId(); ++ let user_name = CStr::from_ptr(pg_sys::GetUserNameFromId(oid, true)); ++ let user_str = user_name.to_str().unwrap(); ++ pg_sys::info!("user: {} trying to change string guc", user_str); ++ if pg_sys::superuser() || user_str == "neon_superuser" || user_str == "neondb_owner" { ++ return true; ++ } ++ pg_sys::ereport!( ++ PgLogLevel::ERROR, ++ PgSqlErrorCode::ERRCODE_INSUFFICIENT_PRIVILEGE, ++ "You are not authorized to change this GUC" ++ ); ++ false ++ } ++} + + // Register the GUC parameters for the extension + // + pub fn register_gucs() { +- GucRegistry::define_string_guc( +- "anon.dummy_locale", +- "The default locale for the dummy data functions", +- "", +- &ANON_DUMMY_LOCALE, +- GucContext::Suset, +- GucFlags::SUPERUSER_ONLY, +- ); +- +- GucRegistry::define_string_guc( +- "anon.k_anonymity_provider", +- "The security label provider used for k-anonymity", +- "", +- &ANON_K_ANONYMITY_PROVIDER, +- GucContext::Suset, +- GucFlags::SUPERUSER_ONLY, +- ); +- +- // +- // As of PGRX 0.12, GUC_LIST_INPUT is not supported which means this +- // parameter can't be properly handled by `SHOW anon.masking_policies` or +- // in the pg_settings catalog. And SplitGUCList has a really weird +- // behaviour with `anon.masking_policies` ¯\_(ツ)_/¯ +- // +- // https://github.com/pgcentralfoundation/pgrx/commit/d096efe6fb2d86e87d117b520b9ccd2f90b2e0d1 +- // +- GucRegistry::define_string_guc( +- "anon.masking_policies", +- "Define additional masking policies (the 'anon' policy is already defined)", +- "", +- &ANON_MASKING_POLICIES, +- GucContext::Suset, +- GucFlags::SUPERUSER_ONLY, /* | GucFlags::LIST_INPUT */ +- ); +- +- GucRegistry::define_bool_guc( +- "anon.privacy_by_default", +- "Mask all columns with NULL (or the default value for NOT NULL columns)", +- "", +- &ANON_PRIVACY_BY_DEFAULT, +- GucContext::Suset, +- GucFlags::default(), +- ); +- GucRegistry::define_bool_guc( +- "anon.transparent_dynamic_masking", +- "New masking engine (EXPERIMENTAL)", +- "", +- &ANON_TRANSPARENT_DYNAMIC_MASKING, +- GucContext::Suset, +- GucFlags::default(), +- ); +- +- GucRegistry::define_bool_guc( +- "anon.restrict_to_trusted_schemas", +- "Masking filters must be in a trusted schema", +- "Activate this option to prevent non-superuser from using their own masking filters", +- &ANON_RESTRICT_TO_TRUSTED_SCHEMAS, +- GucContext::Suset, +- GucFlags::SUPERUSER_ONLY, +- ); +- +- GucRegistry::define_bool_guc( +- "anon.strict_mode", +- "A masking rule cannot change a column data type, unless you disable this", +- "Disabling the mode is not recommended", +- &ANON_STRICT_MODE, +- GucContext::Suset, +- GucFlags::default(), +- ); +- +- // The GUC vars below are not used in the Rust code +- // but they are used in the plpgsql code +- +- GucRegistry::define_string_guc( +- "anon.algorithm", +- "The hash method used for pseudonymizing functions", +- "", +- &ANON_ALGORITHM, +- GucContext::Suset, +- GucFlags::SUPERUSER_ONLY, +- ); +- +- GucRegistry::define_string_guc( +- "anon.maskschema", +- "The schema where the dynamic masking views are stored", +- "", +- &ANON_MASK_SCHEMA, +- GucContext::Suset, +- GucFlags::default(), +- ); +- +- GucRegistry::define_string_guc( +- "anon.salt", +- "The salt value used for the pseudonymizing functions", +- "", +- &ANON_SALT, +- GucContext::Suset, +- GucFlags::SUPERUSER_ONLY, +- ); +- +- GucRegistry::define_string_guc( +- "anon.sourceschema", +- "The schema where the table are masked by the dynamic masking engine", +- "", +- &ANON_SOURCE_SCHEMA, +- GucContext::Suset, +- GucFlags::default(), +- ); ++ unsafe { ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.dummy_locale", ++ c"The default locale for the dummy data functions", ++ c"", ++ &ANON_DUMMY_LOCALE, ++ GucContext::Suset, ++ GucFlags::SUPERUSER_ONLY, ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.k_anonymity_provider", ++ c"The security label provider used for k-anonymity", ++ c"", ++ &ANON_K_ANONYMITY_PROVIDER, ++ GucContext::Suset, ++ GucFlags::SUPERUSER_ONLY, ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ ++ // ++ // As of PGRX 0.12, GUC_LIST_INPUT is not supported which means this ++ // parameter can't be properly handled by `SHOW anon.masking_policies` or ++ // in the pg_settings catalog. And SplitGUCList has a really weird ++ // behaviour with `anon.masking_policies` ¯\_(ツ)_/¯ ++ // ++ // https://github.com/pgcentralfoundation/pgrx/commit/d096efe6fb2d86e87d117b520b9ccd2f90b2e0d1 ++ // ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.masking_policies", ++ c"Define additional masking policies (the 'anon' policy is already defined)", ++ c"", ++ &ANON_MASKING_POLICIES, ++ GucContext::Suset, ++ GucFlags::SUPERUSER_ONLY, /* | GucFlags::LIST_INPUT */ ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_bool_guc_with_hooks( ++ c"anon.privacy_by_default", ++ c"Mask all columns with NULL (or the default value for NOT NULL columns)", ++ c"", ++ &ANON_PRIVACY_BY_DEFAULT, ++ GucContext::Userset, ++ GucFlags::default(), ++ Some(check_bool_guc_hook), ++ None, ++ None, ++ ); ++ GucRegistry::define_bool_guc_with_hooks( ++ c"anon.transparent_dynamic_masking", ++ c"New masking engine (EXPERIMENTAL)", ++ c"", ++ &ANON_TRANSPARENT_DYNAMIC_MASKING, ++ GucContext::Userset, ++ GucFlags::default(), ++ Some(check_bool_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_bool_guc_with_hooks( ++ c"anon.restrict_to_trusted_schemas", ++ c"Masking filters must be in a trusted schema", ++ c"Activate this option to prevent non-superuser from using their own masking filters", ++ &ANON_RESTRICT_TO_TRUSTED_SCHEMAS, ++ GucContext::Suset, ++ GucFlags::SUPERUSER_ONLY, ++ Some(check_bool_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_bool_guc_with_hooks( ++ c"anon.strict_mode", ++ c"A masking rule cannot change a column data type, unless you disable this", ++ c"Disabling the mode is not recommended", ++ &ANON_STRICT_MODE, ++ GucContext::Userset, ++ GucFlags::default(), ++ Some(check_bool_guc_hook), ++ None, ++ None, ++ ); ++ ++ // The GUC vars below are not used in the Rust code ++ // but they are used in the plpgsql code ++ ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.algorithm", ++ c"The hash method used for pseudonymizing functions", ++ c"", ++ &ANON_ALGORITHM, ++ GucContext::Suset, ++ GucFlags::SUPERUSER_ONLY, ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.maskschema", ++ c"The schema where the dynamic masking views are stored", ++ c"", ++ &ANON_MASK_SCHEMA, ++ GucContext::Userset, ++ GucFlags::default(), ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.salt", ++ c"The salt value used for the pseudonymizing functions", ++ c"", ++ &ANON_SALT, ++ GucContext::Suset, ++ GucFlags::SUPERUSER_ONLY, ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ ++ GucRegistry::define_string_guc_with_hooks( ++ c"anon.sourceschema", ++ c"The schema where the table are masked by the dynamic masking engine", ++ c"", ++ &ANON_SOURCE_SCHEMA, ++ GucContext::Userset, ++ GucFlags::default(), ++ Some(check_string_guc_hook), ++ None, ++ None, ++ ); ++ } + } +diff --git a/src/label_providers.rs b/src/label_providers.rs +index d1f406c..30774d7 100644 +--- a/src/label_providers.rs ++++ b/src/label_providers.rs +@@ -36,7 +36,7 @@ pub fn register_label_providers() { + + // Register the default masking policy and the user-defined masking policies + for policy_str in masking::list_masking_policies() { +- let policy_cstring: CString = CString::new(policy_str).unwrap(); ++ let policy_cstring: CString = CString::new(policy_str.clone()).unwrap(); + let policy_ptr: *const c_char = policy_cstring.as_ptr(); + unsafe { + log::debug1!("Anon: registering masking policy '{}'", policy_str); +diff --git a/src/lib.rs b/src/lib.rs +index dd03927..aca37c4 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -429,7 +429,7 @@ mod anon { + + #[cfg(debug_assertions)] + #[pg_extern] +- pub fn list_masking_policies() -> Vec<&'static str> { ++ pub fn list_masking_policies() -> Vec { + masking::list_masking_policies() + } + +diff --git a/src/masking.rs b/src/masking.rs +index cfd2151..998ecce 100644 +--- a/src/masking.rs ++++ b/src/masking.rs +@@ -42,8 +42,8 @@ pub fn get_masking_policy(roleid: pg_sys::Oid) -> Option { + // also the roles that the user belongs to + // This may be done by using `roles_is_member_of()` ? + for policy in list_masking_policies() { +- if has_mask_in_policy(roleid, policy) { +- return Some(policy.to_string()); ++ if has_mask_in_policy(roleid, &policy) { ++ return Some(policy); + } + } + +@@ -59,13 +59,13 @@ pub fn get_masking_policy(roleid: pg_sys::Oid) -> Option { + /// approach (spaces are not handled) and we use `:` as separator to avoid + /// confusion with traditional GUC_LIST_QUOTE parameters. + /// +-pub fn list_masking_policies() -> Vec<&'static str> { ++pub fn list_masking_policies() -> Vec { + use crate::label_providers::ANON_DEFAULT_MASKING_POLICY; + +- let mut masking_policies = vec![ANON_DEFAULT_MASKING_POLICY]; ++ let mut masking_policies = vec![ANON_DEFAULT_MASKING_POLICY.to_string()]; + masking_policies.append(&mut re::capture_guc_list( +- guc::ANON_MASKING_POLICIES.get().unwrap(), +- )); ++ guc::ANON_MASKING_POLICIES.get().unwrap().as_c_str(), ++ ).into_iter().map(String::from).collect()); + masking_policies + } + +@@ -390,7 +390,7 @@ fn generation_expressions(relid: pg_sys::Oid) -> String { + + /// Check that a role is masked in the given policy + /// +-fn has_mask_in_policy(roleid: pg_sys::Oid, policy: &'static str) -> bool { ++fn has_mask_in_policy(roleid: pg_sys::Oid, policy: &str) -> bool { + if let Ok(seclabel) = rule_on_role(roleid, policy) { + return re::is_match_masked(seclabel); + } diff --git a/patches/anon_dyn_mask.patch b/patches/anon_dyn_mask.patch new file mode 100644 index 0000000..4faf927 --- /dev/null +++ b/patches/anon_dyn_mask.patch @@ -0,0 +1,136 @@ +diff --git a/sql/anon.sql b/sql/anon.sql +index 0cdc769..b450327 100644 +--- a/sql/anon.sql ++++ b/sql/anon.sql +@@ -1141,3 +1141,15 @@ $$ + -- TODO : https://en.wikipedia.org/wiki/L-diversity + + -- TODO : https://en.wikipedia.org/wiki/T-closeness ++ ++-- NEON Patches ++ ++GRANT ALL ON SCHEMA anon to neon_superuser; ++GRANT ALL ON ALL TABLES IN SCHEMA anon TO neon_superuser; ++ ++DO $$ ++BEGIN ++ IF current_setting('server_version_num')::int >= 150000 THEN ++ GRANT SET ON PARAMETER anon.transparent_dynamic_masking TO neon_superuser; ++ END IF; ++END $$; +diff --git a/sql/init.sql b/sql/init.sql +index 7da6553..9b6164b 100644 +--- a/sql/init.sql ++++ b/sql/init.sql +@@ -74,50 +74,49 @@ $$ + + SECURITY LABEL FOR anon ON FUNCTION anon.load_csv IS 'UNTRUSTED'; + +--- load fake data from a given path +-CREATE OR REPLACE FUNCTION anon.init( +- datapath TEXT +-) ++CREATE OR REPLACE FUNCTION anon.load_fake_data() + RETURNS BOOLEAN + AS $$ + DECLARE +- datapath_check TEXT; + success BOOLEAN; ++ sharedir TEXT; ++ datapath TEXT; + BEGIN + +- IF anon.is_initialized() THEN +- RAISE NOTICE 'The anon extension is already initialized.'; +- RETURN TRUE; +- END IF; ++ datapath := '/extension/anon/'; ++ -- find the local extension directory ++ SELECT setting INTO sharedir ++ FROM pg_catalog.pg_config ++ WHERE name = 'SHAREDIR'; + + SELECT bool_or(results) INTO success + FROM unnest(array[ +- anon.load_csv('anon.identifiers_category',datapath||'/identifiers_category.csv'), +- anon.load_csv('anon.identifier',datapath ||'/identifier.csv'), +- anon.load_csv('anon.address',datapath ||'/address.csv'), +- anon.load_csv('anon.city',datapath ||'/city.csv'), +- anon.load_csv('anon.company',datapath ||'/company.csv'), +- anon.load_csv('anon.country',datapath ||'/country.csv'), +- anon.load_csv('anon.email', datapath ||'/email.csv'), +- anon.load_csv('anon.first_name',datapath ||'/first_name.csv'), +- anon.load_csv('anon.iban',datapath ||'/iban.csv'), +- anon.load_csv('anon.last_name',datapath ||'/last_name.csv'), +- anon.load_csv('anon.postcode',datapath ||'/postcode.csv'), +- anon.load_csv('anon.siret',datapath ||'/siret.csv'), +- anon.load_csv('anon.lorem_ipsum',datapath ||'/lorem_ipsum.csv') ++ anon.load_csv('anon.identifiers_category',sharedir || datapath || '/identifiers_category.csv'), ++ anon.load_csv('anon.identifier',sharedir || datapath || '/identifier.csv'), ++ anon.load_csv('anon.address',sharedir || datapath || '/address.csv'), ++ anon.load_csv('anon.city',sharedir || datapath || '/city.csv'), ++ anon.load_csv('anon.company',sharedir || datapath || '/company.csv'), ++ anon.load_csv('anon.country',sharedir || datapath || '/country.csv'), ++ anon.load_csv('anon.email', sharedir || datapath || '/email.csv'), ++ anon.load_csv('anon.first_name',sharedir || datapath || '/first_name.csv'), ++ anon.load_csv('anon.iban',sharedir || datapath || '/iban.csv'), ++ anon.load_csv('anon.last_name',sharedir || datapath || '/last_name.csv'), ++ anon.load_csv('anon.postcode',sharedir || datapath || '/postcode.csv'), ++ anon.load_csv('anon.siret',sharedir || datapath || '/siret.csv'), ++ anon.load_csv('anon.lorem_ipsum',sharedir || datapath || '/lorem_ipsum.csv') + ]) results; + RETURN success; +- + END; + $$ +- LANGUAGE PLPGSQL ++ LANGUAGE plpgsql + VOLATILE + RETURNS NULL ON NULL INPUT +- PARALLEL UNSAFE -- because load_csv is unsafe +- SECURITY INVOKER ++ PARALLEL UNSAFE -- because of the EXCEPTION ++ SECURITY DEFINER + SET search_path='' + ; +-SECURITY LABEL FOR anon ON FUNCTION anon.init(TEXT) IS 'UNTRUSTED'; ++ ++SECURITY LABEL FOR anon ON FUNCTION anon.load_fake_data IS 'UNTRUSTED'; + + -- People tend to forget the anon.init() step + -- This is a friendly notice for them +@@ -144,7 +143,7 @@ SECURITY LABEL FOR anon ON FUNCTION anon.notice_if_not_init IS 'UNTRUSTED'; + CREATE OR REPLACE FUNCTION anon.load(TEXT) + RETURNS BOOLEAN AS + $$ +- SELECT anon.init($1); ++ SELECT anon.init(); + $$ + LANGUAGE SQL + VOLATILE +@@ -159,16 +158,16 @@ SECURITY LABEL FOR anon ON FUNCTION anon.load(TEXT) IS 'UNTRUSTED'; + CREATE OR REPLACE FUNCTION anon.init() + RETURNS BOOLEAN + AS $$ +- WITH conf AS ( +- -- find the local extension directory +- SELECT setting AS sharedir +- FROM pg_catalog.pg_config +- WHERE name = 'SHAREDIR' +- ) +- SELECT anon.init(conf.sharedir || '/extension/anon/') +- FROM conf; ++BEGIN ++ IF anon.is_initialized() THEN ++ RAISE NOTICE 'The anon extension is already initialized.'; ++ RETURN TRUE; ++ END IF; ++ ++ RETURN anon.load_fake_data(); ++END; + $$ +- LANGUAGE SQL ++ LANGUAGE plpgsql + VOLATILE + PARALLEL UNSAFE -- because init is unsafe + SECURITY INVOKER diff --git a/sql/anon.sql b/sql/anon.sql index 0e377e3..fbcbe63 100644 --- a/sql/anon.sql +++ b/sql/anon.sql @@ -383,7 +383,7 @@ BEGIN RETURN FALSE; END IF; - EXECUTE format(' + EXECUTE pg_catalog.format(' UPDATE %I SET %I = %I * (1+ (2 * random() - 1 ) * %L) ; ', noise_table, noise_column, noise_column, ratio @@ -413,7 +413,7 @@ BEGIN RETURN FALSE; END IF; - EXECUTE format('UPDATE %I SET %I = %I + (2 * random() - 1 ) * ''%s''::INTERVAL', + EXECUTE pg_catalog.format('UPDATE %I SET %I = %I + (2 * random() - 1 ) * ''%s''::INTERVAL', noise_table, noise_column, noise_column, @@ -518,7 +518,7 @@ BEGIN END IF; -- shuffle - EXECUTE format(' + EXECUTE pg_catalog.format(' WITH s1 AS ( -- shuffle the primary key SELECT row_number() over (order by random()) n, @@ -836,7 +836,7 @@ BEGIN FOR r IN SELECT relnamespace, relname, attname FROM anon.pg_masking_rules LOOP - EXECUTE format('SECURITY LABEL FOR anon ON COLUMN %I.%I.%I IS NULL', + EXECUTE pg_catalog.format('SECURITY LABEL FOR anon ON COLUMN %I.%I.%I IS NULL', r.relnamespace, r.relname, r.attname @@ -1047,7 +1047,7 @@ BEGIN RETURN NULL; END IF; - EXECUTE format(E' + EXECUTE pg_catalog.format(E' SELECT min(c) AS k_anonymity FROM ( SELECT COUNT(*) as c @@ -1071,3 +1071,22 @@ $$ -- TODO : https://en.wikipedia.org/wiki/L-diversity -- TODO : https://en.wikipedia.org/wiki/T-closeness + +-- NEON Patches + +GRANT ALL ON SCHEMA anon to neon_superuser; +GRANT ALL ON ALL TABLES IN SCHEMA anon TO neon_superuser; + +DO $$ +DECLARE + privileged_role_name text; +BEGIN + privileged_role_name := pg_catalog.current_setting('neon.privileged_role_name'); + + EXECUTE pg_catalog.format('GRANT ALL ON SCHEMA anon to %I', privileged_role_name); + EXECUTE pg_catalog.format('GRANT ALL ON ALL TABLES IN SCHEMA anon TO %I', privileged_role_name); + + IF pg_catalog.current_setting('server_version_num')::int >= 150000 THEN + EXECUTE pg_catalog.format('GRANT SET ON PARAMETER anon.transparent_dynamic_masking TO %I', privileged_role_name); + END IF; +END $$; diff --git a/sql/init.sql b/sql/init.sql index 7da6553..f10168a 100644 --- a/sql/init.sql +++ b/sql/init.sql @@ -37,7 +37,7 @@ BEGIN IF sequence IS NOT NULL THEN - EXECUTE format( 'SELECT pg_catalog.setval(%L, max(oid)) FROM %s', + EXECUTE pg_catalog.format( 'SELECT pg_catalog.setval(%L, max(oid)) FROM %s', sequence, dest_table ); @@ -74,50 +74,49 @@ $$ SECURITY LABEL FOR anon ON FUNCTION anon.load_csv IS 'UNTRUSTED'; --- load fake data from a given path -CREATE OR REPLACE FUNCTION anon.init( - datapath TEXT -) +CREATE OR REPLACE FUNCTION anon.load_fake_data() RETURNS BOOLEAN AS $$ DECLARE - datapath_check TEXT; success BOOLEAN; + sharedir TEXT; + datapath TEXT; BEGIN - IF anon.is_initialized() THEN - RAISE NOTICE 'The anon extension is already initialized.'; - RETURN TRUE; - END IF; + datapath := '/extension/anon/'; + -- find the local extension directory + SELECT setting INTO sharedir + FROM pg_catalog.pg_config + WHERE name = 'SHAREDIR'; - SELECT bool_or(results) INTO success - FROM unnest(array[ - anon.load_csv('anon.identifiers_category',datapath||'/identifiers_category.csv'), - anon.load_csv('anon.identifier',datapath ||'/identifier.csv'), - anon.load_csv('anon.address',datapath ||'/address.csv'), - anon.load_csv('anon.city',datapath ||'/city.csv'), - anon.load_csv('anon.company',datapath ||'/company.csv'), - anon.load_csv('anon.country',datapath ||'/country.csv'), - anon.load_csv('anon.email', datapath ||'/email.csv'), - anon.load_csv('anon.first_name',datapath ||'/first_name.csv'), - anon.load_csv('anon.iban',datapath ||'/iban.csv'), - anon.load_csv('anon.last_name',datapath ||'/last_name.csv'), - anon.load_csv('anon.postcode',datapath ||'/postcode.csv'), - anon.load_csv('anon.siret',datapath ||'/siret.csv'), - anon.load_csv('anon.lorem_ipsum',datapath ||'/lorem_ipsum.csv') + SELECT pg_catalog.bool_or(results) INTO success + FROM pgcatalog.unnest(array[ + anon.load_csv('anon.identifiers_category',sharedir || datapath || '/identifiers_category.csv'), + anon.load_csv('anon.identifier',sharedir || datapath || '/identifier.csv'), + anon.load_csv('anon.address',sharedir || datapath || '/address.csv'), + anon.load_csv('anon.city',sharedir || datapath || '/city.csv'), + anon.load_csv('anon.company',sharedir || datapath || '/company.csv'), + anon.load_csv('anon.country',sharedir || datapath || '/country.csv'), + anon.load_csv('anon.email', sharedir || datapath || '/email.csv'), + anon.load_csv('anon.first_name',sharedir || datapath || '/first_name.csv'), + anon.load_csv('anon.iban',sharedir || datapath || '/iban.csv'), + anon.load_csv('anon.last_name',sharedir || datapath || '/last_name.csv'), + anon.load_csv('anon.postcode',sharedir || datapath || '/postcode.csv'), + anon.load_csv('anon.siret',sharedir || datapath || '/siret.csv'), + anon.load_csv('anon.lorem_ipsum',sharedir || datapath || '/lorem_ipsum.csv') ]) results; RETURN success; - END; $$ - LANGUAGE PLPGSQL + LANGUAGE plpgsql VOLATILE RETURNS NULL ON NULL INPUT - PARALLEL UNSAFE -- because load_csv is unsafe - SECURITY INVOKER + PARALLEL UNSAFE -- because of the EXCEPTION + SECURITY DEFINER SET search_path='' ; -SECURITY LABEL FOR anon ON FUNCTION anon.init(TEXT) IS 'UNTRUSTED'; + +SECURITY LABEL FOR anon ON FUNCTION anon.load_fake_data IS 'UNTRUSTED'; -- People tend to forget the anon.init() step -- This is a friendly notice for them @@ -144,7 +143,7 @@ SECURITY LABEL FOR anon ON FUNCTION anon.notice_if_not_init IS 'UNTRUSTED'; CREATE OR REPLACE FUNCTION anon.load(TEXT) RETURNS BOOLEAN AS $$ - SELECT anon.init($1); + SELECT anon.init(); $$ LANGUAGE SQL VOLATILE @@ -159,16 +158,16 @@ SECURITY LABEL FOR anon ON FUNCTION anon.load(TEXT) IS 'UNTRUSTED'; CREATE OR REPLACE FUNCTION anon.init() RETURNS BOOLEAN AS $$ - WITH conf AS ( - -- find the local extension directory - SELECT setting AS sharedir - FROM pg_catalog.pg_config - WHERE name = 'SHAREDIR' - ) - SELECT anon.init(conf.sharedir || '/extension/anon/') - FROM conf; +BEGIN + IF anon.is_initialized() THEN + RAISE NOTICE 'The anon extension is already initialized.'; + RETURN TRUE; + END IF; + + RETURN anon.load_fake_data(); +END; $$ - LANGUAGE SQL + LANGUAGE plpgsql VOLATILE PARALLEL UNSAFE -- because init is unsafe SECURITY INVOKER diff --git a/sql/replica_masking.sql b/sql/replica_masking.sql new file mode 100644 index 0000000..6d60af4 --- /dev/null +++ b/sql/replica_masking.sql @@ -0,0 +1,141 @@ + + +CREATE OR REPLACE FUNCTION anon.start_replica_masking( + policy TEXT DEFAULT 'anon' +) +RETURNS BOOLEAN AS +$$ + SELECT anon.refresh_replica_masking(policy) +$$ + LANGUAGE SQL + VOLATILE + PARALLEL UNSAFE -- because of CREATE TRIGGER + SECURITY INVOKER + SET search_path='' +; + +SECURITY LABEL FOR anon ON FUNCTION anon.start_replica_masking(TEXT) IS 'UNTRUSTED'; + +-- Walk through all tables with masked columns and +-- update the replica masking triggers +CREATE OR REPLACE FUNCTION anon.refresh_replica_masking( + policy TEXT DEFAULT 'anon' +) +RETURNS BOOLEAN AS +$$ + SELECT bool_or(anon.refresh_replica_trigger_for_table(t.regclass,policy)) + FROM ( + SELECT distinct attrelid::REGCLASS as regclass + FROM anon.pg_masking_rules + ) as t; +$$ + LANGUAGE SQL + VOLATILE + PARALLEL UNSAFE -- because of CREATE TRIGGER + SECURITY INVOKER + SET search_path='' +; + +SECURITY LABEL FOR anon ON FUNCTION anon.refresh_replica_masking(TEXT) IS 'UNTRUSTED'; + + +-- +-- The Event trigger below works fine (almost). But since the +-- `anon.refresh_replica_trigger_for_table` is not transactionnal the event +-- trigger won't work within a long transaction.... +-- +-- +-- We're keeping this while waiting for a way to remove the SPI calls. +-- +-- -- +-- -- +-- -- This is run after any SECURITY LABEL statement +-- -- +-- CREATE OR REPLACE FUNCTION anon.tg_refresh_replica_masking() +-- RETURNS EVENT_TRIGGER AS +-- $$ +-- DECLARE +-- seclabel_on_column_event RECORD; +-- BEGIN +-- FOR seclabel_on_column_event IN +-- SELECT * +-- FROM pg_catalog.pg_event_trigger_ddl_commands() +-- WHERE classid = 'pg_class'::REGCLASS::OID +-- AND objsubid IS NOT NULL -- The SECLABEL is on a column +-- AND NOT e.in_extension -- Ignore the init script +-- AND pg_catalog.current_setting(anon.replica_masking) +-- LOOP +-- RAISE DEBUG 'Anon: Refreshing replica maskign trigger for %', seclabel_on_column_event.object_identity; +-- PERFORM anon.refresh_replica_trigger_for_table(seclabel_on_column_event.objid,'anon'); +-- END LOOP; +-- END +-- $$ +-- LANGUAGE plpgsql +-- PARALLEL UNSAFE -- because of UPDATE +-- SECURITY INVOKER +-- SET search_path='' +-- ; +-- +-- SECURITY LABEL FOR anon ON FUNCTION anon.tg_refresh_replica_masking IS 'UNTRUSTED'; +-- +-- +-- CREATE EVENT TRIGGER anon_tg_refresh_replica_masking +-- ON ddl_command_end +-- WHEN TAG IN ( 'SECURITY LABEL' ) +-- EXECUTE PROCEDURE anon.tg_refresh_replica_masking() +-- ; + + + +-- Walk through all triggers and function we created previously +-- and drop everything +CREATE OR REPLACE FUNCTION anon.stop_replica_masking() +RETURNS BOOLEAN AS +$$ +DECLARE + trigger RECORD; + function RECORD; +BEGIN + FOR trigger IN + SELECT + t.trigger_name, + t.event_object_table, + t.event_object_schema + FROM information_schema.triggers t + WHERE t.trigger_name LIKE 'tg_anon_replica_masking_%' + LOOP + EXECUTE pg_catalog.format('DROP TRIGGER IF EXISTS %I ON %I.%I', + trigger.trigger_name, + trigger.event_object_schema, + trigger.event_object_table + ); + END LOOP; + + FOR function IN + SELECT + p.proname as function_name, + n.nspname as schema_name, + pg_catalog.pg_get_function_identity_arguments(p.oid) as function_args + FROM pg_proc p + JOIN pg_namespace n ON p.pronamespace = n.oid + WHERE n.nspname = 'anon' + AND p.proname LIKE 'replica%' + LOOP + EXECUTE pg_catalog.format('DROP FUNCTION IF EXISTS %I.%I(%s)', + function.schema_name, + function.function_name, + function.function_args + ); + END LOOP; + + RETURN TRUE; +END; +$$ + LANGUAGE plpgsql + VOLATILE + PARALLEL UNSAFE -- because of DROP TRIGGER + SECURITY INVOKER + SET search_path='' +; + +SECURITY LABEL FOR anon ON FUNCTION anon.stop_replica_masking() IS 'UNTRUSTED'; diff --git a/src/dummy.rs b/src/dummy.rs index a514bb4..e448031 100644 --- a/src/dummy.rs +++ b/src/dummy.rs @@ -106,11 +106,8 @@ macro_rules! declare_l10n_fn_String { #[pg_extern] pub fn $name() -> String { - let locale = $crate::guc::ANON_DUMMY_LOCALE - .get() - .unwrap() - .to_str() - .expect("Should be a string"); + let guc_value = $crate::guc::ANON_DUMMY_LOCALE.get().unwrap(); + let locale = guc_value.to_str().expect("Should be a string"); dummy!($struct, locale) } }; @@ -132,11 +129,8 @@ macro_rules! declare_l10n_fn_with_range_to_string { #[pg_extern] pub fn $name(r: pgrx::Range) -> String { - let locale = $crate::guc::ANON_DUMMY_LOCALE - .get() - .unwrap() - .to_str() - .expect("Should be a string"); + let guc_value = $crate::guc::ANON_DUMMY_LOCALE.get().unwrap(); + let locale = guc_value.to_str().expect("Should be a string"); return $crate::dummy_with_range!($struct, locale, r); } }; diff --git a/src/fixture.rs b/src/fixture.rs index 907adf2..d13e8ee 100644 --- a/src/fixture.rs +++ b/src/fixture.rs @@ -262,6 +262,16 @@ pub fn disable_static_masking() { .unwrap(); } +#[allow(dead_code)] +pub fn enable_replica_masking() { + Spi::run( + " + SET anon.replica_masking TO on; + ", + ) + .unwrap(); +} + #[allow(dead_code)] pub fn trust_masking_functions_schema() { Spi::run( diff --git a/src/guc.rs b/src/guc.rs index e423a3d..2e6b689 100644 --- a/src/guc.rs +++ b/src/guc.rs @@ -3,25 +3,21 @@ //---------------------------------------------------------------------------- use pgrx::*; -use std::ffi::CStr; +use std::ffi::{c_void, CStr, CString}; -pub static ANON_DUMMY_LOCALE: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"en_US\0") - })); +pub static ANON_DUMMY_LOCALE: GucSetting> = + GucSetting::>::new(Some(c"en_US")); -pub static ANON_K_ANONYMITY_PROVIDER: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"k_anonymity\0") - })); +pub static ANON_K_ANONYMITY_PROVIDER: GucSetting> = + GucSetting::>::new(Some(c"k_anonymity")); -pub static ANON_MASKING_POLICIES: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"\0") - })); +pub static ANON_MASKING_POLICIES: GucSetting> = + GucSetting::>::new(Some(c"")); pub static ANON_PRIVACY_BY_DEFAULT: GucSetting = GucSetting::::new(false); +pub static ANON_REPLICA_MASKING: GucSetting = GucSetting::::new(false); + pub static ANON_RESTRICT_TO_TRUSTED_SCHEMAS: GucSetting = GucSetting::::new(true); pub static ANON_STRICT_MODE: GucSetting = GucSetting::::new(true); @@ -33,63 +29,137 @@ pub static ANON_STATIC_MASKING: GucSetting = GucSetting::::new(true) // The GUC vars below are not used in the Rust code // but they are used in the plpgsql code -static ANON_ALGORITHM: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"sha256\0") - })); +static ANON_ALGORITHM: GucSetting> = + GucSetting::>::new(Some(c"sha256")); + +static ANON_SALT: GucSetting> = GucSetting::>::new(Some(c"")); + +static ANON_SOURCE_SCHEMA: GucSetting> = + GucSetting::>::new(Some(c"public")); -static ANON_SALT: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"\0") - })); +static ANON_MASK_SCHEMA: GucSetting> = + GucSetting::>::new(Some(c"mask")); -static ANON_SOURCE_SCHEMA: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"public\0") - })); +unsafe extern "C-unwind" fn check_bool_guc_hook( + _newval: *mut bool, + _extra: *mut *mut c_void, + source: u32, +) -> bool { + unsafe { + // The sources that we allow are: + // 1. PGC_S_DEFAULT (0) -> for default boot up source, likely new session or server. + // 2. PGC_S_DATABASE (6) -> a GUC set for a particular database + // 3. PGC_S_USER (7) -> a GUC set for a particular role + // 4. PGC_S_DATABASE_USER (8) -> a GUC set for a particular role in a particular database + // This check only allows sources that load a variable, not ones that try to alter it. + // Sources that try to alter it are: + // 1. PGC_S_FILE (3) -> ALTER SYSTEM + // 2. PGC_S_TEST (12) -> ALTER ROLE/DATABASE + // 3. PGC_S_SESSION (13) -> SET ... + // TODO (thesuhas): Does PGC_S_GLOBAL need to be added to whitelisted sources? + pg_sys::info!("Source: {}", source); + if source == 0 || source == 6 || source == 7 || source == 8 { + return true; + } + let oid = pg_sys::GetUserId(); + let user_name = CStr::from_ptr(pg_sys::GetUserNameFromId(oid, true)); + let user_str = user_name.to_str().unwrap(); + pg_sys::info!("user: {} trying to change boolean guc", user_str); + if pg_sys::superuser() || user_str == "neon_superuser" || user_str == "neondb_owner" { + return true; + } + pg_sys::ereport!( + PgLogLevel::ERROR, + PgSqlErrorCode::ERRCODE_INSUFFICIENT_PRIVILEGE, + "You are not authorized to change this GUC" + ); + false + } +} -static ANON_MASK_SCHEMA: GucSetting> = - GucSetting::>::new(Some(unsafe { - CStr::from_bytes_with_nul_unchecked(b"mask\0") - })); +unsafe extern "C-unwind" fn check_string_guc_hook( + _newval: *mut *mut libc::c_char, + _extra: *mut *mut c_void, + source: u32, +) -> bool { + unsafe { + // The sources that we allow are: + // 1. PGC_S_DEFAULT (0) -> for default boot up source, likely new session or server. + // 2. PGC_S_DATABASE (6) -> a GUC set for a particular database + // 3. PGC_S_USER (7) -> a GUC set for a particular role + // 4. PGC_S_DATABASE_USER (8) -> a GUC set for a particular role in a particular database + // This check only allows sources that load a variable, not ones that try to alter it. + // Sources that try to alter it are: + // 1. PGC_S_FILE (3) -> ALTER SYSTEM + // 2. PGC_S_TEST (12) -> ALTER ROLE/DATABASE + // 3. PGC_S_SESSION (13) -> SET ... + pg_sys::info!("Source: {}", source); + if source == 0 || source == 6 || source == 7 || source == 8 { + return true; + } + let oid = pg_sys::GetUserId(); + let user_name = CStr::from_ptr(pg_sys::GetUserNameFromId(oid, true)); + let user_str = user_name.to_str().unwrap(); + pg_sys::info!("user: {} trying to change string guc", user_str); + if pg_sys::superuser() || user_str == "neon_superuser" || user_str == "neondb_owner" { + return true; + } + pg_sys::ereport!( + PgLogLevel::ERROR, + PgSqlErrorCode::ERRCODE_INSUFFICIENT_PRIVILEGE, + "You are not authorized to change this GUC" + ); + false + } +} // Register the GUC parameters for the extension // pub fn register_gucs() { - GucRegistry::define_string_guc( - "anon.dummy_locale", - "The default locale for the dummy data functions", - "", - &ANON_DUMMY_LOCALE, - GucContext::Suset, - GucFlags::SUPERUSER_ONLY, - ); + unsafe { + GucRegistry::define_string_guc_with_hooks( + c"anon.dummy_locale", + c"The default locale for the dummy data functions", + c"", + &ANON_DUMMY_LOCALE, + GucContext::Suset, + GucFlags::SUPERUSER_ONLY, + Some(check_string_guc_hook), + None, + None, + ); - GucRegistry::define_string_guc( - "anon.k_anonymity_provider", - "The security label provider used for k-anonymity", - "", - &ANON_K_ANONYMITY_PROVIDER, - GucContext::Suset, - GucFlags::SUPERUSER_ONLY, - ); + GucRegistry::define_string_guc_with_hooks( + c"anon.k_anonymity_provider", + c"The security label provider used for k-anonymity", + c"", + &ANON_K_ANONYMITY_PROVIDER, + GucContext::Suset, + GucFlags::SUPERUSER_ONLY, + Some(check_string_guc_hook), + None, + None, + ); - // - // As of PGRX 0.12, GUC_LIST_INPUT is not supported which means this - // parameter can't be properly handled by `SHOW anon.masking_policies` or - // in the pg_settings catalog. And SplitGUCList has a really weird - // behaviour with `anon.masking_policies` ¯\_(ツ)_/¯ - // - // https://github.com/pgcentralfoundation/pgrx/commit/d096efe6fb2d86e87d117b520b9ccd2f90b2e0d1 - // - GucRegistry::define_string_guc( - "anon.masking_policies", - "Define additional masking policies (the 'anon' policy is already defined)", - "", - &ANON_MASKING_POLICIES, - GucContext::Suset, - GucFlags::SUPERUSER_ONLY, /* | GucFlags::LIST_INPUT */ - ); + // + // As of PGRX 0.12, GUC_LIST_INPUT is not supported which means this + // parameter can't be properly handled by `SHOW anon.masking_policies` or + // in the pg_settings catalog. And SplitGUCList has a really weird + // behaviour with `anon.masking_policies` ¯\_(ツ)_/¯ + // + // https://github.com/pgcentralfoundation/pgrx/commit/d096efe6fb2d86e87d117b520b9ccd2f90b2e0d1 + // + GucRegistry::define_string_guc_with_hooks( + c"anon.masking_policies", + c"Define additional masking policies (the 'anon' policy is already defined)", + c"", + &ANON_MASKING_POLICIES, + GucContext::Suset, + GucFlags::SUPERUSER_ONLY, /* | GucFlags::LIST_INPUT */ + Some(check_string_guc_hook), + None, + None, + ); GucRegistry::define_bool_guc( "anon.privacy_by_default", @@ -117,6 +187,15 @@ pub fn register_gucs() { GucFlags::default(), ); + GucRegistry::define_bool_guc( + "anon.replica_masking", + "Masking a logical replica (EXPERIMENTAL)", + "", + &ANON_REPLICA_MASKING, + GucContext::Suset, + GucFlags::default(), + ); + GucRegistry::define_bool_guc( "anon.restrict_to_trusted_schemas", "Masking filters must be in a trusted schema", @@ -126,51 +205,67 @@ pub fn register_gucs() { GucFlags::SUPERUSER_ONLY, ); - GucRegistry::define_bool_guc( - "anon.strict_mode", - "A masking rule cannot change a column data type, unless you disable this", - "Disabling the mode is not recommended", - &ANON_STRICT_MODE, - GucContext::Suset, - GucFlags::default(), - ); + GucRegistry::define_bool_guc_with_hooks( + c"anon.strict_mode", + c"A masking rule cannot change a column data type, unless you disable this", + c"Disabling the mode is not recommended", + &ANON_STRICT_MODE, + GucContext::Userset, + GucFlags::default(), + Some(check_bool_guc_hook), + None, + None, + ); - // The GUC vars below are not used in the Rust code - // but they are used in the plpgsql code + // The GUC vars below are not used in the Rust code + // but they are used in the plpgsql code - GucRegistry::define_string_guc( - "anon.algorithm", - "The hash method used for pseudonymizing functions", - "", - &ANON_ALGORITHM, - GucContext::Suset, - GucFlags::SUPERUSER_ONLY, - ); + GucRegistry::define_string_guc_with_hooks( + c"anon.algorithm", + c"The hash method used for pseudonymizing functions", + c"", + &ANON_ALGORITHM, + GucContext::Suset, + GucFlags::SUPERUSER_ONLY, + Some(check_string_guc_hook), + None, + None, + ); - GucRegistry::define_string_guc( - "anon.maskschema", - "The schema where the dynamic masking views are stored", - "", - &ANON_MASK_SCHEMA, - GucContext::Suset, - GucFlags::default(), - ); + GucRegistry::define_string_guc_with_hooks( + c"anon.maskschema", + c"The schema where the dynamic masking views are stored", + c"", + &ANON_MASK_SCHEMA, + GucContext::Userset, + GucFlags::default(), + Some(check_string_guc_hook), + None, + None, + ); - GucRegistry::define_string_guc( - "anon.salt", - "The salt value used for the pseudonymizing functions", - "", - &ANON_SALT, - GucContext::Suset, - GucFlags::SUPERUSER_ONLY, - ); + GucRegistry::define_string_guc_with_hooks( + c"anon.salt", + c"The salt value used for the pseudonymizing functions", + c"", + &ANON_SALT, + GucContext::Suset, + GucFlags::SUPERUSER_ONLY, + Some(check_string_guc_hook), + None, + None, + ); - GucRegistry::define_string_guc( - "anon.sourceschema", - "The schema where the table are masked by the dynamic masking engine", - "", - &ANON_SOURCE_SCHEMA, - GucContext::Suset, - GucFlags::default(), - ); + GucRegistry::define_string_guc_with_hooks( + c"anon.sourceschema", + c"The schema where the table are masked by the dynamic masking engine", + c"", + &ANON_SOURCE_SCHEMA, + GucContext::Userset, + GucFlags::default(), + Some(check_string_guc_hook), + None, + None, + ); + } } diff --git a/src/label_providers.rs b/src/label_providers.rs index d1f406c..30774d7 100644 --- a/src/label_providers.rs +++ b/src/label_providers.rs @@ -36,7 +36,7 @@ pub fn register_label_providers() { // Register the default masking policy and the user-defined masking policies for policy_str in masking::list_masking_policies() { - let policy_cstring: CString = CString::new(policy_str).unwrap(); + let policy_cstring: CString = CString::new(policy_str.clone()).unwrap(); let policy_ptr: *const c_char = policy_cstring.as_ptr(); unsafe { log::debug1!("Anon: registering masking policy '{}'", policy_str); diff --git a/src/lib.rs b/src/lib.rs index 85dfd54..ce4fd8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ mod macros; mod masking; mod random; mod re; +mod replica_masking; mod sampling; mod static_masking; mod utils; @@ -40,6 +41,8 @@ extension_sql_file!("../sql/pseudo.sql", requires = ["init"]); extension_sql_file!("../sql/random.sql", requires = ["anon"]); extension_sql_file!("../sql/static_masking.sql", requires = ["anon"]); extension_sql_file!("../sql/legacy_dynamic_masking.sql", requires = ["anon"]); +extension_sql_file!("../sql/replica_masking.sql", requires = ["anon"]); + // GCOVR_EXCL_STOP pgrx::pg_module_magic!(); @@ -323,6 +326,33 @@ mod anon { requires = ["anon"] ); + //------------------------------------------------------------------------ + // Replica Masking + //------------------------------------------------------------------------ + use crate::replica_masking; + + #[pg_extern] + pub fn drop_replica_trigger_for_table(r: pg_sys::Oid) -> Option { + replica_masking::drop_replica_trigger_for_table(r) + } + + #[pg_extern] + pub fn refresh_replica_trigger_for_table(r: pg_sys::Oid, p: String) -> Option { + replica_masking::refresh_replica_trigger_for_table(r, p) + } + + // + // The replica masking functions should not be used as masking filters + // + extension_sql!( + r#" + SECURITY LABEL FOR anon ON FUNCTION anon.drop_replica_trigger_for_table(OID) IS 'UNTRUSTED'; + SECURITY LABEL FOR anon ON FUNCTION anon.refresh_replica_trigger_for_table(OID,TEXT) IS 'UNTRUSTED'; + "#, + name = "unstrust_replica_masking_functions", + requires = ["anon"] + ); + //------------------------------------------------------------------------ // Masking engine //------------------------------------------------------------------------ @@ -430,7 +460,7 @@ mod anon { #[cfg(debug_assertions)] #[pg_extern] - pub fn list_masking_policies() -> Vec<&'static str> { + pub fn list_masking_policies() -> Vec { masking::list_masking_policies() } diff --git a/src/masking.rs b/src/masking.rs index cfd2151..998ecce 100644 --- a/src/masking.rs +++ b/src/masking.rs @@ -42,8 +42,8 @@ pub fn get_masking_policy(roleid: pg_sys::Oid) -> Option { // also the roles that the user belongs to // This may be done by using `roles_is_member_of()` ? for policy in list_masking_policies() { - if has_mask_in_policy(roleid, policy) { - return Some(policy.to_string()); + if has_mask_in_policy(roleid, &policy) { + return Some(policy); } } @@ -59,13 +59,13 @@ pub fn get_masking_policy(roleid: pg_sys::Oid) -> Option { /// approach (spaces are not handled) and we use `:` as separator to avoid /// confusion with traditional GUC_LIST_QUOTE parameters. /// -pub fn list_masking_policies() -> Vec<&'static str> { +pub fn list_masking_policies() -> Vec { use crate::label_providers::ANON_DEFAULT_MASKING_POLICY; - let mut masking_policies = vec![ANON_DEFAULT_MASKING_POLICY]; + let mut masking_policies = vec![ANON_DEFAULT_MASKING_POLICY.to_string()]; masking_policies.append(&mut re::capture_guc_list( - guc::ANON_MASKING_POLICIES.get().unwrap(), - )); + guc::ANON_MASKING_POLICIES.get().unwrap().as_c_str(), + ).into_iter().map(String::from).collect()); masking_policies } @@ -390,7 +390,7 @@ fn generation_expressions(relid: pg_sys::Oid) -> String { /// Check that a role is masked in the given policy /// -fn has_mask_in_policy(roleid: pg_sys::Oid, policy: &'static str) -> bool { +fn has_mask_in_policy(roleid: pg_sys::Oid, policy: &str) -> bool { if let Ok(seclabel) = rule_on_role(roleid, policy) { return re::is_match_masked(seclabel); } diff --git a/src/random.rs b/src/random.rs index bbbd99d..23be812 100644 --- a/src/random.rs +++ b/src/random.rs @@ -56,15 +56,27 @@ fn range_usize(r: Range) -> Option> { }) } -/// Convert a pgrx::Range into a Rust Range:: +/// Convert a pgrx::Range into a Rust Range:: /// /!\ unbounded range are not allowed -fn range_usize_from_i64(r: Range) -> Option> { +fn range_isize(r: Range) -> Option> { if r.is_infinite() { return None; } - Some(core::ops::Range:: { - start: *r.lower()?.get()? as usize, - end: *r.upper()?.get()? as usize, + Some(core::ops::Range:: { + start: *r.lower()?.get()? as isize, + end: *r.upper()?.get()? as isize, + }) +} + +/// Convert a pgrx::Range into a Rust Range:: +/// /!\ unbounded range are not allowed +fn range_isize_from_i64(r: Range) -> Option> { + if r.is_infinite() { + return None; + } + Some(core::ops::Range:: { + start: *r.lower()?.get()? as isize, + end: *r.upper()?.get()? as isize, }) } @@ -122,7 +134,7 @@ pub fn time() -> pgrx::datum::Time { //---------------------------------------------------------------------------- pub fn bigint(r: Range) -> Option { - Some(i64::try_from(range_usize_from_i64(r)?.fake::()).expect("Out of Bound")) + Some(i64::try_from(range_isize_from_i64(r)?.fake::()).expect("Out of Bound")) } pub fn double_precision(r: Range) -> Option { @@ -130,7 +142,7 @@ pub fn double_precision(r: Range) -> Option { } pub fn int(r: Range) -> Option { - Some(i32::try_from(range_usize(r)?.fake::()).expect("Out of Bound")) + Some(i32::try_from(range_isize(r)?.fake::()).expect("Out of Bound")) } pub fn number_with_format(format: String) -> String { @@ -168,6 +180,7 @@ mod tests { #[pg_test] fn test_int() { assert!(int(pgrx::Range::::new(1, 10)).is_some()); + assert!(int(pgrx::Range::::new(-10, -1)).is_some()); assert_eq!(int(pgrx::Range::::new(1, 2)), Some(1)); assert!(int(pgrx::Range::::new(None, 10)).is_none()); assert!(int(pgrx::Range::::new(1, None)).is_none()); @@ -185,6 +198,7 @@ mod tests { #[pg_test] fn test_bigint() { assert!(bigint(pgrx::Range::::new(1, 10)).is_some()); + assert!(bigint(pgrx::Range::::new(-10, -1)).is_some()); assert!(bigint(pgrx::Range::::new(None, 10)).is_none()); assert!(bigint(pgrx::Range::::new(1, None)).is_none()); assert!(bigint(pgrx::Range::::new(None, None)).is_none()); @@ -196,7 +210,10 @@ mod tests { let two = pgrx::AnyNumeric::from(2); let six = pgrx::AnyNumeric::from(6); let ten = pgrx::AnyNumeric::from(10); + let minus_one = pgrx::AnyNumeric::from(-1); + let minus_ten = pgrx::AnyNumeric::from(-10); assert!(range_f32(pgrx::Range::::new(one, ten)).is_some()); + assert!(range_f32(pgrx::Range::::new(minus_one, minus_ten)).is_some()); assert!(range_f32(pgrx::Range::::new(None, six)).is_none()); assert!(range_f32(pgrx::Range::::new(two, None)).is_none()); assert!(double_precision(pgrx::Range::::new(None, None)).is_none()); @@ -208,7 +225,10 @@ mod tests { let two = pgrx::AnyNumeric::from(2); let six = pgrx::AnyNumeric::from(6); let ten = pgrx::AnyNumeric::from(10); + let minus_one = pgrx::AnyNumeric::from(-1); + let minus_ten = pgrx::AnyNumeric::from(-10); assert!(range_f64(pgrx::Range::::new(one, ten)).is_some()); + assert!(range_f64(pgrx::Range::::new(minus_one, minus_ten)).is_some()); assert!(range_f64(pgrx::Range::::new(None, six)).is_none()); assert!(range_f64(pgrx::Range::::new(two, None)).is_none()); assert!(range_f64(pgrx::Range::::new(None, None)).is_none()); @@ -220,7 +240,12 @@ mod tests { let two = pgrx::AnyNumeric::from(2); let six = pgrx::AnyNumeric::from(6); let ten = pgrx::AnyNumeric::from(10); + let minus_one = pgrx::AnyNumeric::from(-1); + let minus_ten = pgrx::AnyNumeric::from(-10); assert!(double_precision(pgrx::Range::::new(one, ten)).is_some()); + assert!( + double_precision(pgrx::Range::::new(minus_ten, minus_one)).is_some() + ); assert!(double_precision(pgrx::Range::::new(None, six)).is_none()); assert!(double_precision(pgrx::Range::::new(two, None)).is_none()); assert!(double_precision(pgrx::Range::::new(None, None)).is_none()); @@ -232,7 +257,10 @@ mod tests { let two = pgrx::AnyNumeric::from(2); let six = pgrx::AnyNumeric::from(6); let ten = pgrx::AnyNumeric::from(10); + let minus_one = pgrx::AnyNumeric::from(-1); + let minus_ten = pgrx::AnyNumeric::from(-10); assert!(numeric(pgrx::Range::::new(one, ten)).is_some()); + assert!(numeric(pgrx::Range::::new(minus_ten, minus_one)).is_some()); assert!(numeric(pgrx::Range::::new(None, six)).is_none()); assert!(numeric(pgrx::Range::::new(two, None)).is_none()); assert!(numeric(pgrx::Range::::new(None, None)).is_none()); @@ -244,7 +272,10 @@ mod tests { let two = pgrx::AnyNumeric::from(2); let six = pgrx::AnyNumeric::from(6); let ten = pgrx::AnyNumeric::from(10); + let minus_one = pgrx::AnyNumeric::from(-1); + let minus_ten = pgrx::AnyNumeric::from(-10); assert!(real(pgrx::Range::::new(one, ten)).is_some()); + assert!(real(pgrx::Range::::new(minus_ten, minus_one)).is_some()); assert!(real(pgrx::Range::::new(None, six)).is_none()); assert!(real(pgrx::Range::::new(two, None)).is_none()); assert!(real(pgrx::Range::::new(None, None)).is_none()); diff --git a/src/replica_masking.rs b/src/replica_masking.rs new file mode 100644 index 0000000..39bda5f --- /dev/null +++ b/src/replica_masking.rs @@ -0,0 +1,198 @@ +/// +/// # Replica Masking +/// +use crate::guc; +use crate::log; +use crate::masking; +use crate::utils; +use fastrand; +use pgrx::prelude::*; + +/// Return the SQL assignments which will mask the data in a trigger +/// +/// ## Example: +/// +/// * if a column `fk_user` is masked with `pg_catalog.md5(fk_user)` +/// * the assignment will look like this +/// +/// NEW.fk_user = (SELECT CAST(pg_catalog.md5(fk_user) AS text) FROM (SELECT NEW.* ) AS n); +/// + +fn trigger_new_assignments(relid: pg_sys::Oid, policy: String) -> Option { + let lockmode = pg_sys::AccessShareLock as i32; + + // SAFETY: `pg_sys::relation_open()` will raise XX000 if the specified oid + // isn't a valid relation + let relation = unsafe { PgBox::from_pg(pg_sys::relation_open(relid, lockmode)) }; + + // reldesc is a TupleDescData object + // https://doxygen.postgresql.org/structTupleDescData.html + let reldesc = unsafe { PgBox::from_pg(relation.rd_att) }; + let natts = reldesc.natts; + let attrs = unsafe { reldesc.attrs.as_slice(natts.try_into().unwrap()) }; + + let mut assignments = Vec::new(); + for a in attrs { + if a.attisdropped { + continue; + } + + let (filter_value, att_is_masked) = masking::value_for_att(&relation, a, policy.clone()); + + // Typically in a for a NEW assignment (INSERT or UPDATE), + // we only want to overwrite the value of the masked columns + if att_is_masked { + assignments.push(format!( + "NEW.{:?} = (SELECT {} FROM (SELECT NEW.* ) AS n);", + name_data_to_str(&a.attname), + filter_value + )); + } + } + + // pass the relation back to Postgres + unsafe { + pg_sys::relation_close(relation.as_ptr(), lockmode); + } + + if assignments.is_empty() { + return None; + } + Some(assignments.join(" ").to_string()) +} + +/// Remove the masking trigger from a table +pub fn drop_replica_trigger_for_table(relid: pg_sys::Oid) -> Option { + let tablename = utils::get_relation_qualified_name(relid)?; + let relint: u32 = relid.into(); + + let sql: String = format_args!( + include_str!("templates/sql/drop_replica_trigger.sql"), + relint = relint, + tablename = tablename, + ) + .to_string(); + + log::debug1!("Anon: {sql}"); + Spi::run(&sql) + .unwrap_or_else(|_| panic!("Failed to drop replica masking triggers for {tablename}")); + + Some(true) +} + +/// Create the masking trigger for a given table in a given policy +/// +/// IMPORTANT: this function is not transactionnal ! +/// +/// The spi::run call will open a new session which means that when the +/// refresh is launched within a transaction, all objects created previously +/// in that transaction (especially tables and security labels) are NOT visible +/// yet for the `create_replica_trigger` script. +/// +pub fn refresh_replica_trigger_for_table(relid: pg_sys::Oid, policy: String) -> Option { + if !guc::ANON_REPLICA_MASKING.get() { + return Some(false); + } + + let tablename = utils::get_relation_qualified_name(relid)?; + + let sql: String = format_args!( + include_str!("templates/sql/create_replica_trigger.sql"), + relint = relid, + tablename = tablename, + new_assignments = trigger_new_assignments(relid, policy.clone())?, + random = fastrand::u8(..), + ) + .to_string(); + + log::debug1!("Anon: {sql}"); + + Spi::run(&sql) + .unwrap_or_else(|_| panic!("Failed to refresh replica masking triggers for {tablename}")); + + Some(true) +} + +//---------------------------------------------------------------------------- +// Tests +//---------------------------------------------------------------------------- + +#[cfg(any(test, feature = "pg_test"))] +#[pg_schema] +mod tests { + use crate::fixture; + use crate::label_providers::ANON_DEFAULT_MASKING_POLICY; + use crate::replica_masking::*; + + #[pg_test] + fn test_trigger_new_assignments() { + let anon = ANON_DEFAULT_MASKING_POLICY.to_string(); + let relid = fixture::create_table_person(); + fixture::enable_replica_masking(); + assert_eq!( + trigger_new_assignments(relid, "does_not_exits".into()), + None + ); + assert_eq!( + trigger_new_assignments(relid, anon.clone()), + Some( + "NEW.\"lastname\" = (SELECT CAST(NULL AS text) FROM (SELECT NEW.* ) AS n);".into() + ) + ); + } + + #[pg_test] + fn test_trigger_new_assignments_with_quotes() { + let anon = ANON_DEFAULT_MASKING_POLICY.to_string(); + let relid = fixture::create_table_user(); + fixture::enable_replica_masking(); + assert_eq!( + trigger_new_assignments(relid, anon), + Some( + "NEW.\"Email\" = (SELECT CAST(anon.fake_email() AS text) FROM (SELECT NEW.* ) AS n);" + .into() + ) + ); + } + + #[pg_test] + fn test_replica_masking_not_enabled() { + let anon = ANON_DEFAULT_MASKING_POLICY.to_string(); + let relid = fixture::create_table_user(); + assert_eq!(Some(false), refresh_replica_trigger_for_table(relid, anon)); + } + + #[pg_test] + fn test_refresh_replica_trigger_for_table_no_policy() { + let policy = "does_not_exist".to_string(); + let relid = fixture::create_table_user(); + fixture::enable_replica_masking(); + assert_eq!(None, refresh_replica_trigger_for_table(relid, policy)); + } + + #[pg_test] + fn test_refresh_replica_trigger_for_table() { + let anon = ANON_DEFAULT_MASKING_POLICY.to_string(); + let relid = fixture::create_table_user(); + fixture::enable_replica_masking(); + assert_eq!(Some(true), refresh_replica_trigger_for_table(relid, anon)); + } + + #[pg_test] + fn test_refresh_replica_trigger_for_table_does_not_exist() { + let anon = ANON_DEFAULT_MASKING_POLICY.to_string(); + fixture::enable_replica_masking(); + assert_eq!( + None, + refresh_replica_trigger_for_table(pg_sys::InvalidOid, anon) + ); + } + + #[pg_test] + fn test_refresh_replica_trigger_for_table_no_rules() { + let anon = ANON_DEFAULT_MASKING_POLICY.to_string(); + let relid = fixture::create_table_location(); + fixture::enable_replica_masking(); + assert_eq!(None, refresh_replica_trigger_for_table(relid, anon.clone())); + } +} diff --git a/src/templates/sql/create_replica_trigger.sql b/src/templates/sql/create_replica_trigger.sql new file mode 100644 index 0000000..e3badf3 --- /dev/null +++ b/src/templates/sql/create_replica_trigger.sql @@ -0,0 +1,54 @@ +-- +-- # Create a masking replica trigger for given table +-- +-- ## Mandatory input parameters : +-- +-- * `relint`: the table oid +-- * `tablename`: the schema-qualified table name (e.g. "MyApp"."MyTable" ) +-- * `new_assignements` : the list of NEW value trigger will change +-- * `random`: a bit of entropy to avoid SQL injections +-- +-- ## Example: +-- +-- For a table named `foo` and oid 546458 with : +-- * a `fk_user` column masked with its md5 hash +-- * a `login` column masked with "CONFIDENTIAL" +-- +-- then the trigger would look like this +-- +-- CREATE OR REPLACE FUNCTION anon.replica_masking_546458() +-- RETURNS TRIGGER AS $func_483187318799514$ +-- BEGIN +-- RAISE DEBUG '% Trigger fired on %.% with % => %', +-- TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME, OLD, NEW; +-- NEW.fk_user = (SELECT CAST(pg_catalog.md5(fk_user) AS text) FROM (SELECT NEW.* ) AS n); +-- NEW.login = (SELECT CAST("CONFIDENTIAL") AS text); +-- RETURN NEW; +-- END; +-- $func_483187318799514$ +-- LANGUAGE plpgsql; +-- +-- + +-- This script may produce NOTICE messages that would be confusing for the user +SET client_min_messages TO WARNING; + +CREATE OR REPLACE FUNCTION anon.replica_masking_{relint}() +RETURNS TRIGGER AS $func_{random}$ +BEGIN + RAISE DEBUG '% Trigger fired on %.% with % => %', + TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME, OLD, NEW; + {new_assignments} +RETURN NEW; +END; +$func_{random}$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS tg_anon_replica_masking_{relint} ON {tablename}; + +CREATE TRIGGER tg_anon_replica_masking_{relint} + BEFORE INSERT OR UPDATE ON {tablename} + FOR EACH ROW + EXECUTE FUNCTION anon.replica_masking_{relint}(); + +ALTER TABLE {tablename} ENABLE REPLICA TRIGGER tg_anon_replica_masking_{relint}; diff --git a/src/templates/sql/drop_replica_trigger.sql b/src/templates/sql/drop_replica_trigger.sql new file mode 100644 index 0000000..8ae55eb --- /dev/null +++ b/src/templates/sql/drop_replica_trigger.sql @@ -0,0 +1,15 @@ +-- +-- # Drop the masking replica trigger for given table +-- +-- ## Mandatory input parameters : +-- +-- * `relint`: the table oid +-- * `tablename`: the schema-qualified table name (e.g. "MyApp"."MyTable" ) +-- + +-- This may produce NOTICE messages that would be confusing for the user +SET client_min_messages TO WARNING; + +DROP TRIGGER IF EXISTS tg_anon_replica_masking_{relint} ON {tablename} CASCADE; + +DROP FUNCTION IF EXISTS anon.replica_masking_{relint}() CASCADE; diff --git a/tests/expected/ldm.out b/tests/expected/ldm.out index 39f78e3..87d45bc 100644 --- a/tests/expected/ldm.out +++ b/tests/expected/ldm.out @@ -84,7 +84,7 @@ BEGIN END$$; NOTICE: insufficient_privilege -- Jimmy can't see GUC_SUPERUSER_ONLY settings -SELECT COUNT(name)=6 FROM pg_settings WHERE name LIKE 'anon.%'; +SELECT COUNT(name)=0 FROM pg_settings WHERE name = 'anon.salt'; ?column? ---------- t @@ -92,7 +92,7 @@ SELECT COUNT(name)=6 FROM pg_settings WHERE name LIKE 'anon.%'; RESET ROLE; -- Super user sees all settings -SELECT COUNT(name)>6 FROM pg_settings WHERE name LIKE 'anon.%'; +SELECT COUNT(name)=1 FROM pg_settings WHERE name = 'anon.salt'; ?column? ---------- t diff --git a/tests/expected/test_replica_masking.out b/tests/expected/test_replica_masking.out new file mode 100644 index 0000000..cc7edb2 --- /dev/null +++ b/tests/expected/test_replica_masking.out @@ -0,0 +1,239 @@ +-- +-- Replica Masking is quite hard to test with pg_regress ! +-- +-- Here's we choose to setup 2 database on the same instance. This is rather +-- unusual configuration but it allows us to run commands from outside the +-- current contrib_regression database +-- +-- When the test fails, you might have to clean up manually the source database +-- with : +-- +-- psql contrib_regression -c 'ALTER SUBSCRIPTION contrib_sub DISABLE' +-- psql contrib_regression -c 'ALTER SUBSCRIPTION contrib_sub SET (slot_name = NONE)' +-- psql contrib_regression -c 'DROP SUBSCRIPTION contrib_sub'; +-- +-- Of course, we can't run this test in a single transaction +-- +CREATE ROLE contrib_repli LOGIN REPLICATION; +------------------------------------------------------------------------------- +-- Setup the source database +------------------------------------------------------------------------------- +CREATE DATABASE contrib_regression_source; +\! psql contrib_regression_source -c 'CREATE SCHEMA "MyApp";' +CREATE SCHEMA +\! psql contrib_regression_source -c 'CREATE TABLE "MyApp".person( id SERIAL PRIMARY KEY, name TEXT, company TEXT );' +CREATE TABLE +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (1, 'Alice', 'CompanyA');" +INSERT 0 1 +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (2, 'Bob', 'CompanyB');" +INSERT 0 1 +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (3, 'Charlie', 'CompanyC');" +INSERT 0 1 +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (4, 'David', 'CompanyD');" +INSERT 0 1 +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (5, 'Eve', 'CompanyE');" +INSERT 0 1 +\! psql contrib_regression_source -c 'GRANT USAGE ON SCHEMA "MyApp" TO contrib_repli;' +GRANT +\! psql contrib_regression_source -c 'GRANT SELECT ON ALL TABLES IN SCHEMA "MyApp" TO contrib_repli;' +GRANT +-- Creating a subscription that connects to the same database cluster will only +-- succeed if the replication slot is not created as part of the same command. +-- Otherwise, the CREATE SUBSCRIPTION call will hang forever. +\! psql contrib_regression_source -c "SELECT slot_name FROM pg_create_logical_replication_slot('contrib_slot', 'pgoutput');" + slot_name +-------------- + contrib_slot +(1 row) + +-- In CI, the command above may take a few seconds, leading to hanging jobs +-- We pause for a while to be safe +SELECT pg_sleep(10); + pg_sleep +---------- + +(1 row) + +\! psql contrib_regression_source -c 'CREATE PUBLICATION contrib_pub FOR TABLE "MyApp".person'; +CREATE PUBLICATION +------------------------------------------------------------------------------- +-- Setup the replica +------------------------------------------------------------------------------- +CREATE EXTENSION anon; +CREATE SCHEMA "MyApp"; +CREATE TABLE "MyApp".person ( + id SERIAL PRIMARY KEY, + name TEXT, + company TEXT +); +GRANT USAGE ON SCHEMA "MyApp" TO contrib_repli; +GRANT ALL ON ALL TABLES IN SCHEMA "MyApp" TO contrib_repli; +SECURITY LABEL FOR anon ON COLUMN "MyApp".person.company + IS 'MASKED WITH FUNCTION pg_catalog.md5(company)'; +-- This should fail +SELECT anon.start_replica_masking(); + start_replica_masking +----------------------- + f +(1 row) + +SET anon.replica_masking TO on; +SELECT anon.start_replica_masking(); + start_replica_masking +----------------------- + t +(1 row) + +SELECT COUNT(*)=1 +FROM pg_trigger +WHERE tgname LIKE 'tg_anon_replica_%'; + ?column? +---------- + t +(1 row) + +-- Replication starts now +CREATE SUBSCRIPTION contrib_sub +CONNECTION 'host=localhost user=contrib_repli password=x dbname=contrib_regression_source' +PUBLICATION contrib_pub +WITH (slot_name='contrib_slot', create_slot=false); +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) + +-- The data is replicated +SELECT name = 'Alice' FROM "MyApp".person WHERE id=1; + ?column? +---------- + t +(1 row) + +-- The masking rules are applied +SELECT company = pg_catalog.md5('CompanyA') FROM "MyApp".person WHERE id=1; + ?column? +---------- + t +(1 row) + +------------------------------------------------------------------------------- +-- Add a new masking rule +------------------------------------------------------------------------------- +SECURITY LABEL FOR anon ON COLUMN "MyApp".person.name + IS 'MASKED WITH FUNCTION anon.dummy_name()'; +SELECT anon.refresh_replica_masking(); + refresh_replica_masking +------------------------- + t +(1 row) + +------------------------------------------------------------------------------- +-- Testing INSERT +------------------------------------------------------------------------------- +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person(id,name, company) VALUES (999999,'John','CompanyF');" +INSERT 0 1 +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) + +SELECT name != 'John' +FROM "MyApp".person +WHERE id=999999; + ?column? +---------- + t +(1 row) + +SELECT company=md5('CompanyF') +FROM "MyApp".person +WHERE id=999999; + ?column? +---------- + t +(1 row) + +------------------------------------------------------------------------------- +-- DELETE +------------------------------------------------------------------------------- +\! psql contrib_regression_source -c 'DELETE FROM "MyApp".person WHERE id=1;' +DELETE 1 +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) + +SELECT COUNT(*)=0 FROM "MyApp".person WHERE id=1; + ?column? +---------- + t +(1 row) + +------------------------------------------------------------------------------- +-- UPDATE +------------------------------------------------------------------------------- +\! psql contrib_regression_source -c "UPDATE \"MyApp\".person SET name='Omar' WHERE company='CompanyB';" +UPDATE 1 +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) + +SELECT name != 'Omar' AND name != 'Bob' +FROM "MyApp".person +WHERE id=2; + ?column? +---------- + t +(1 row) + +------------------------------------------------------------------------------- +-- Stop Replica Masking +------------------------------------------------------------------------------- +SELECT anon.stop_replica_masking(); + stop_replica_masking +---------------------- + t +(1 row) + +SELECT COUNT(*)=0 +FROM pg_trigger +WHERE tgname LIKE 'tg_anon_replica_%'; + ?column? +---------- + t +(1 row) + +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person(id,name,company) VALUES (88,'Alfred', 'CompanyG');" +INSERT 0 1 +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + pg_sleep +---------- + +(1 row) + +SELECT name = 'Alfred' AND company = 'CompanyG' +FROM "MyApp".person +WHERE id=88; + ?column? +---------- + t +(1 row) + +-- Clean up +DROP SCHEMA "MyApp" CASCADE; +DROP SUBSCRIPTION contrib_sub; +\! psql contrib_regression_source -c 'DROP PUBLICATION contrib_pub;' +DROP PUBLICATION +DROP DATABASE contrib_regression_source; +DROP ROLE contrib_repli; +DROP EXTENSION anon CASCADE; diff --git a/tests/sql/ldm.sql b/tests/sql/ldm.sql index 3c2ebae..2c48249 100644 --- a/tests/sql/ldm.sql +++ b/tests/sql/ldm.sql @@ -82,12 +82,12 @@ BEGIN END$$; -- Jimmy can't see GUC_SUPERUSER_ONLY settings -SELECT COUNT(name)=6 FROM pg_settings WHERE name LIKE 'anon.%'; +SELECT COUNT(name)=0 FROM pg_settings WHERE name = 'anon.salt'; RESET ROLE; -- Super user sees all settings -SELECT COUNT(name)>6 FROM pg_settings WHERE name LIKE 'anon.%'; +SELECT COUNT(name)=1 FROM pg_settings WHERE name = 'anon.salt'; -- Bug #259 - anon should not interact with other extensions CREATE EXTENSION pg_stat_statements; diff --git a/tests/sql/test_replica_masking.sql b/tests/sql/test_replica_masking.sql new file mode 100644 index 0000000..4b92431 --- /dev/null +++ b/tests/sql/test_replica_masking.sql @@ -0,0 +1,183 @@ +-- +-- Replica Masking is quite hard to test with pg_regress ! +-- +-- Here's we choose to setup 2 database on the same instance. This is rather +-- unusual configuration but it allows us to run commands from outside the +-- current contrib_regression database +-- +-- When the test fails, you might have to clean up manually the source database +-- with : +-- +-- psql contrib_regression -c 'ALTER SUBSCRIPTION contrib_sub DISABLE' +-- psql contrib_regression -c 'ALTER SUBSCRIPTION contrib_sub SET (slot_name = NONE)' +-- psql contrib_regression -c 'DROP SUBSCRIPTION contrib_sub'; +-- +-- Of course, we can't run this test in a single transaction +-- + +CREATE ROLE contrib_repli LOGIN REPLICATION; + +------------------------------------------------------------------------------- +-- Setup the source database +------------------------------------------------------------------------------- + +CREATE DATABASE contrib_regression_source; + +\! psql contrib_regression_source -c 'CREATE SCHEMA "MyApp";' + +\! psql contrib_regression_source -c 'CREATE TABLE "MyApp".person( id SERIAL PRIMARY KEY, name TEXT, company TEXT );' + +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (1, 'Alice', 'CompanyA');" +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (2, 'Bob', 'CompanyB');" +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (3, 'Charlie', 'CompanyC');" +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (4, 'David', 'CompanyD');" +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person VALUES (5, 'Eve', 'CompanyE');" + + +\! psql contrib_regression_source -c 'GRANT USAGE ON SCHEMA "MyApp" TO contrib_repli;' +\! psql contrib_regression_source -c 'GRANT SELECT ON ALL TABLES IN SCHEMA "MyApp" TO contrib_repli;' + + +-- Creating a subscription that connects to the same database cluster will only +-- succeed if the replication slot is not created as part of the same command. +-- Otherwise, the CREATE SUBSCRIPTION call will hang forever. +\! psql contrib_regression_source -c "SELECT slot_name FROM pg_create_logical_replication_slot('contrib_slot', 'pgoutput');" + +-- In CI, the command above may take a few seconds, leading to hanging jobs +-- We pause for a while to be safe +SELECT pg_sleep(10); + +\! psql contrib_regression_source -c 'CREATE PUBLICATION contrib_pub FOR TABLE "MyApp".person'; + + +------------------------------------------------------------------------------- +-- Setup the replica +------------------------------------------------------------------------------- + +CREATE EXTENSION anon; + +CREATE SCHEMA "MyApp"; + +CREATE TABLE "MyApp".person ( + id SERIAL PRIMARY KEY, + name TEXT, + company TEXT +); + +GRANT USAGE ON SCHEMA "MyApp" TO contrib_repli; +GRANT ALL ON ALL TABLES IN SCHEMA "MyApp" TO contrib_repli; + + +SECURITY LABEL FOR anon ON COLUMN "MyApp".person.company + IS 'MASKED WITH FUNCTION pg_catalog.md5(company)'; + +-- This should fail +SELECT anon.start_replica_masking(); + +SET anon.replica_masking TO on; + +SELECT anon.start_replica_masking(); + +SELECT COUNT(*)=1 +FROM pg_trigger +WHERE tgname LIKE 'tg_anon_replica_%'; + +-- Replication starts now +CREATE SUBSCRIPTION contrib_sub +CONNECTION 'host=localhost user=contrib_repli password=x dbname=contrib_regression_source' +PUBLICATION contrib_pub +WITH (slot_name='contrib_slot', create_slot=false); + +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + +-- The data is replicated +SELECT name = 'Alice' FROM "MyApp".person WHERE id=1; + +-- The masking rules are applied +SELECT company = pg_catalog.md5('CompanyA') FROM "MyApp".person WHERE id=1; + +------------------------------------------------------------------------------- +-- Add a new masking rule +------------------------------------------------------------------------------- + +SECURITY LABEL FOR anon ON COLUMN "MyApp".person.name + IS 'MASKED WITH FUNCTION anon.dummy_name()'; + +SELECT anon.refresh_replica_masking(); + +------------------------------------------------------------------------------- +-- Testing INSERT +------------------------------------------------------------------------------- + +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person(id,name, company) VALUES (999999,'John','CompanyF');" + +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + +SELECT name != 'John' +FROM "MyApp".person +WHERE id=999999; + +SELECT company=md5('CompanyF') +FROM "MyApp".person +WHERE id=999999; + + +------------------------------------------------------------------------------- +-- DELETE +------------------------------------------------------------------------------- + +\! psql contrib_regression_source -c 'DELETE FROM "MyApp".person WHERE id=1;' + +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + +SELECT COUNT(*)=0 FROM "MyApp".person WHERE id=1; + +------------------------------------------------------------------------------- +-- UPDATE +------------------------------------------------------------------------------- + +\! psql contrib_regression_source -c "UPDATE \"MyApp\".person SET name='Omar' WHERE company='CompanyB';" + +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + +SELECT name != 'Omar' AND name != 'Bob' +FROM "MyApp".person +WHERE id=2; + +------------------------------------------------------------------------------- +-- Stop Replica Masking +------------------------------------------------------------------------------- + +SELECT anon.stop_replica_masking(); + +SELECT COUNT(*)=0 +FROM pg_trigger +WHERE tgname LIKE 'tg_anon_replica_%'; + +\! psql contrib_regression_source -c "INSERT INTO \"MyApp\".person(id,name,company) VALUES (88,'Alfred', 'CompanyG');" + + +-- Synchronisation may a take a few milliseconds +SELECT pg_sleep(2); + +SELECT name = 'Alfred' AND company = 'CompanyG' +FROM "MyApp".person +WHERE id=88; + +-- Clean up + +DROP SCHEMA "MyApp" CASCADE; + +DROP SUBSCRIPTION contrib_sub; + +\! psql contrib_regression_source -c 'DROP PUBLICATION contrib_pub;' + +DROP DATABASE contrib_regression_source; + +DROP ROLE contrib_repli; + +DROP EXTENSION anon CASCADE;