From 50d1354c0a2f23bbe98798be479d8d8c66807c31 Mon Sep 17 00:00:00 2001 From: 06kellyjac Date: Fri, 16 Feb 2024 16:41:38 +0000 Subject: [PATCH] semgrep: 1.37.0 -> 1.61.1 Updated references to return2corp github org to the semgrep github org Resolved issues new broken tests or skipped --- pkgs/tools/security/semgrep/common.nix | 20 +++++----- pkgs/tools/security/semgrep/default.nix | 40 +++++++++++++++++++- pkgs/tools/security/semgrep/semgrep-core.nix | 2 +- pkgs/tools/security/semgrep/update.sh | 8 ++-- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/pkgs/tools/security/semgrep/common.nix b/pkgs/tools/security/semgrep/common.nix index da646461a20f..e4e51d81e9b7 100644 --- a/pkgs/tools/security/semgrep/common.nix +++ b/pkgs/tools/security/semgrep/common.nix @@ -1,9 +1,9 @@ { lib }: rec { - version = "1.37.0"; + version = "1.61.1"; - srcHash = "sha256-oFJ43dq3DAhux0UEFDKFZnxruoRdOfCndKY6XgG3d5I="; + srcHash = "sha256-muTw6rj9FuSSXvUzdP4QRQogzmUPlrvGARRK/Jqg+Gc="; # submodule dependencies # these are fetched so we: @@ -11,10 +11,10 @@ rec { # 2. avoid fetchSubmodules since it's prone to impurities submodules = { "cli/src/semgrep/semgrep_interfaces" = { - owner = "returntocorp"; + owner = "semgrep"; repo = "semgrep-interfaces"; - rev = "331603197022625f50a64dd5e3029a96a5f03ada"; - hash = "sha256-UAcWbTSCIdBGvgGSbdQ+miFOEuBvQ6m42MkU3VeErKY="; + rev = "bbfd1c5b91bd411bceffc3de73f5f0b37f04433d"; + hash = "sha256-wrhV5bBuIpVYehzVTxussiED//ObJXQSfPiiKnIR/DM="; }; }; @@ -25,22 +25,22 @@ rec { core = { x86_64-linux = { platform = "any"; - hash = "sha256-Sj/6tzZMyRQAJL09X/3zgvdGTIhNibqO8usKsus9Xss="; + hash = "sha256-lX/zRgkEyoln69pf4fWtb8f9wffBOI/KkCegn8kFmj4="; }; x86_64-darwin = { platform = "macosx_10_14_x86_64"; - hash = "sha256-hC04VknZG6aYYNX7lqvkcOoVslewNqlYax+o1nV2TcM="; + hash = "sha256-Rk4qP/iKpRUbqdry6V/NmXRQLkA0e9ltIOdYiO5DuTg="; }; aarch64-darwin = { platform = "macosx_11_0_arm64"; - hash = "sha256-0F+ndM4+0dnxf9acwWvGdIy9iYWSqixS9IzOxa95/yM="; + hash = "sha256-Gqq9LGwZ96i8LU8Z8qSN3TxuUUTDYrJiVCY9rm7aNzI="; }; }; meta = with lib; { homepage = "https://semgrep.dev/"; - downloadPage = "https://github.com/returntocorp/semgrep/"; - changelog = "https://github.com/returntocorp/semgrep/blob/v${version}/CHANGELOG.md"; + downloadPage = "https://github.com/semgrep/semgrep/"; + changelog = "https://github.com/semgrep/semgrep/blob/v${version}/CHANGELOG.md"; description = "Lightweight static analysis for many languages"; longDescription = '' Semgrep is a fast, open-source, static analysis tool for finding bugs and diff --git a/pkgs/tools/security/semgrep/default.nix b/pkgs/tools/security/semgrep/default.nix index ff41daacac30..70e6b8641ee8 100644 --- a/pkgs/tools/security/semgrep/default.nix +++ b/pkgs/tools/security/semgrep/default.nix @@ -1,5 +1,6 @@ { lib , fetchFromGitHub +, fetchpatch , semgrep-core , buildPythonApplication , pythonPackages @@ -9,19 +10,31 @@ , git }: +# testing locally post build: +# ./result/bin/semgrep scan --metrics=off --config 'r/generic.unicode.security.bidi.contains-bidirectional-characters' + let common = import ./common.nix { inherit lib; }; + semgrepBinPath = lib.makeBinPath [ semgrep-core ]; in buildPythonApplication rec { pname = "semgrep"; inherit (common) version; src = fetchFromGitHub { - owner = "returntocorp"; + owner = "semgrep"; repo = "semgrep"; rev = "v${version}"; hash = common.srcHash; }; + patches = [ + (fetchpatch { + name = "fix-test_dump_engine-test-for-nix-store-path.patch"; + url = "https://github.com/semgrep/semgrep/commit/c7553c1a61251146773617f80a2d360e6b6ab3f9.patch"; + hash = "sha256-A3QdL0DDh/pbDpRIBACUie7PEvC17iG4t6qTnmPIwA4="; + }) + ]; + # prepare a subset of the submodules as we only need a handful # and there are many many submodules total postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList @@ -72,34 +85,57 @@ buildPythonApplication rec { ]; doCheck = true; + nativeCheckInputs = [ git pytestCheckHook ] ++ (with pythonPackages; [ + flaky pytest-snapshot pytest-mock pytest-freezegun types-freezegun ]); + disabledTests = [ # requires networking "test_send" # requires networking "test_parse_exclude_rules_auto" + # many child tests require networking to download files + "TestConfigLoaderForProducts" + # doesn't start flaky plugin correctly + "test_debug_performance" ]; + preCheck = '' # tests need a home directory export HOME="$(mktemp -d)" + # tests need access to `semgrep-core` + export OLD_PATH="$PATH" + export PATH="$PATH:${semgrepBinPath}" + + # we're in cli + # replace old semgrep with wrapped one + rm ./bin/semgrep + ln -s $out/bin/semgrep ./bin/semgrep + # disabledTestPaths doesn't manage to avoid the e2e tests # remove them from pyproject.toml # and remove need for pytest-split substituteInPlace pyproject.toml \ --replace '"tests/e2e",' "" \ + --replace '"tests/e2e-pro",' "" \ --replace 'addopts = "--splitting-algorithm=least_duration"' "" ''; + postCheck = '' + export PATH="$OLD_PATH" + unset OLD_PATH + ''; + # since we stop cli/setup.py from finding semgrep-core and copying it into # the result we need to provide it on the PATH preFixup = '' - makeWrapperArgs+=(--prefix PATH : ${lib.makeBinPath [ semgrep-core ]}) + makeWrapperArgs+=(--prefix PATH : ${semgrepBinPath}) ''; postInstall = '' diff --git a/pkgs/tools/security/semgrep/semgrep-core.nix b/pkgs/tools/security/semgrep/semgrep-core.nix index b924dd16579f..33e50837bf75 100644 --- a/pkgs/tools/security/semgrep/semgrep-core.nix +++ b/pkgs/tools/security/semgrep/semgrep-core.nix @@ -20,7 +20,7 @@ stdenvNoCC.mkDerivation rec { inherit version; format = "wheel"; dist = python; - python = "cp37.cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311"; + python = "cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311"; inherit (data) platform hash; }; diff --git a/pkgs/tools/security/semgrep/update.sh b/pkgs/tools/security/semgrep/update.sh index c66180cf8f2b..67b720154b0b 100755 --- a/pkgs/tools/security/semgrep/update.sh +++ b/pkgs/tools/security/semgrep/update.sh @@ -24,10 +24,10 @@ instantiateClean() { # get latest version NEW_VERSION=$( - curl -s -H \ + curl -s -L -H \ "Accept: application/vnd.github.v3+json" \ ${GITHUB_TOKEN:+ -H "Authorization: bearer $GITHUB_TOKEN"} \ - https://api.github.com/repos/returntocorp/semgrep/releases/latest \ + https://api.github.com/repos/semgrep/semgrep/releases/latest \ | jq -r '.tag_name' ) # trim v prefix @@ -58,7 +58,7 @@ fetchPypi rec { version = \"$VERSION\"; format = \"wheel\"; dist = python; - python = \"cp37.cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311\"; + python = \"cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311\"; platform = \"$PLATFORM\"; } " @@ -101,7 +101,7 @@ update_core_platform "aarch64-darwin" OLD_PWD=$PWD TMPDIR="$(mktemp -d)" # shallow clone to check submodule commits, don't actually need the submodules -git clone https://github.com/returntocorp/semgrep "$TMPDIR/semgrep" --depth 1 --branch "v$NEW_VERSION" +git clone https://github.com/semgrep/semgrep "$TMPDIR/semgrep" --depth 1 --branch "v$NEW_VERSION" get_submodule_commit() { OLD_PWD=$PWD