Merge pull request #289339 from 06kellyjac/semgrep

semgrep{,-core}: 1.37.0 -> 1.61.1

pkgs/tools/security/semgrep/common.nix

@@ -1,9 +1,9 @@
 { lib }:
 
 rec {
-  version = "1.37.0";
+  version = "1.61.1";
 
-  srcHash = "sha256-oFJ43dq3DAhux0UEFDKFZnxruoRdOfCndKY6XgG3d5I=";
+  srcHash = "sha256-muTw6rj9FuSSXvUzdP4QRQogzmUPlrvGARRK/Jqg+Gc=";
 
   # submodule dependencies
   # these are fetched so we:
@@ -11,10 +11,10 @@ rec {
   #   2. avoid fetchSubmodules since it's prone to impurities
   submodules = {
     "cli/src/semgrep/semgrep_interfaces" = {
-      owner = "returntocorp";
+      owner = "semgrep";
       repo = "semgrep-interfaces";
-      rev = "331603197022625f50a64dd5e3029a96a5f03ada";
-      hash = "sha256-UAcWbTSCIdBGvgGSbdQ+miFOEuBvQ6m42MkU3VeErKY=";
+      rev = "bbfd1c5b91bd411bceffc3de73f5f0b37f04433d";
+      hash = "sha256-wrhV5bBuIpVYehzVTxussiED//ObJXQSfPiiKnIR/DM=";
     };
   };
 
@@ -25,22 +25,22 @@ rec {
   core = {
     x86_64-linux = {
       platform = "any";
-      hash = "sha256-Sj/6tzZMyRQAJL09X/3zgvdGTIhNibqO8usKsus9Xss=";
+      hash = "sha256-lX/zRgkEyoln69pf4fWtb8f9wffBOI/KkCegn8kFmj4=";
     };
     x86_64-darwin = {
       platform = "macosx_10_14_x86_64";
-      hash = "sha256-hC04VknZG6aYYNX7lqvkcOoVslewNqlYax+o1nV2TcM=";
+      hash = "sha256-Rk4qP/iKpRUbqdry6V/NmXRQLkA0e9ltIOdYiO5DuTg=";
     };
     aarch64-darwin = {
       platform = "macosx_11_0_arm64";
-      hash = "sha256-0F+ndM4+0dnxf9acwWvGdIy9iYWSqixS9IzOxa95/yM=";
+      hash = "sha256-Gqq9LGwZ96i8LU8Z8qSN3TxuUUTDYrJiVCY9rm7aNzI=";
     };
   };
 
   meta = with lib; {
     homepage = "https://semgrep.dev/";
-    downloadPage = "https://github.com/returntocorp/semgrep/";
-    changelog = "https://github.com/returntocorp/semgrep/blob/v${version}/CHANGELOG.md";
+    downloadPage = "https://github.com/semgrep/semgrep/";
+    changelog = "https://github.com/semgrep/semgrep/blob/v${version}/CHANGELOG.md";
     description = "Lightweight static analysis for many languages";
     longDescription = ''
       Semgrep is a fast, open-source, static analysis tool for finding bugs and

pkgs/tools/security/semgrep/default.nix

@@ -1,5 +1,6 @@
 { lib
 , fetchFromGitHub
+, fetchpatch
 , semgrep-core
 , buildPythonApplication
 , pythonPackages
@@ -9,19 +10,31 @@
 , git
 }:
 
+# testing locally post build:
+# ./result/bin/semgrep scan --metrics=off --config 'r/generic.unicode.security.bidi.contains-bidirectional-characters'
+
 let
   common = import ./common.nix { inherit lib; };
+  semgrepBinPath = lib.makeBinPath [ semgrep-core ];
 in
 buildPythonApplication rec {
   pname = "semgrep";
   inherit (common) version;
   src = fetchFromGitHub {
-    owner = "returntocorp";
+    owner = "semgrep";
     repo = "semgrep";
     rev = "v${version}";
     hash = common.srcHash;
   };
 
+  patches = [
+    (fetchpatch {
+      name = "fix-test_dump_engine-test-for-nix-store-path.patch";
+      url = "https://github.com/semgrep/semgrep/commit/c7553c1a61251146773617f80a2d360e6b6ab3f9.patch";
+      hash = "sha256-A3QdL0DDh/pbDpRIBACUie7PEvC17iG4t6qTnmPIwA4=";
+    })
+  ];
+
   # prepare a subset of the submodules as we only need a handful
   # and there are many many submodules total
   postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList
@@ -72,34 +85,57 @@ buildPythonApplication rec {
   ];
 
   doCheck = true;
+
   nativeCheckInputs = [ git pytestCheckHook ] ++ (with pythonPackages; [
+    flaky
     pytest-snapshot
     pytest-mock
     pytest-freezegun
     types-freezegun
   ]);
+
   disabledTests = [
     # requires networking
     "test_send"
     # requires networking
     "test_parse_exclude_rules_auto"
+    # many child tests require networking to download files
+    "TestConfigLoaderForProducts"
+    # doesn't start flaky plugin correctly
+    "test_debug_performance"
   ];
+
   preCheck = ''
     # tests need a home directory
     export HOME="$(mktemp -d)"
 
+    # tests need access to `semgrep-core`
+    export OLD_PATH="$PATH"
+    export PATH="$PATH:${semgrepBinPath}"
+
+    # we're in cli
+    # replace old semgrep with wrapped one
+    rm ./bin/semgrep
+    ln -s $out/bin/semgrep ./bin/semgrep
+
     # disabledTestPaths doesn't manage to avoid the e2e tests
     # remove them from pyproject.toml
     # and remove need for pytest-split
     substituteInPlace pyproject.toml \
       --replace '"tests/e2e",' "" \
+      --replace '"tests/e2e-pro",' "" \
       --replace 'addopts = "--splitting-algorithm=least_duration"' ""
   '';
 
+  postCheck = ''
+    export PATH="$OLD_PATH"
+    unset OLD_PATH
+  '';
+
   # since we stop cli/setup.py from finding semgrep-core and copying it into
   # the result we need to provide it on the PATH
   preFixup = ''
-    makeWrapperArgs+=(--prefix PATH : ${lib.makeBinPath [ semgrep-core ]})
+    makeWrapperArgs+=(--prefix PATH : ${semgrepBinPath})
   '';
 
   postInstall = ''

pkgs/tools/security/semgrep/semgrep-core.nix

@@ -20,7 +20,7 @@ stdenvNoCC.mkDerivation rec {
       inherit version;
       format = "wheel";
       dist = python;
-      python = "cp37.cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311";
+      python = "cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311";
       inherit (data) platform hash;
     };
 

pkgs/tools/security/semgrep/update.sh

@@ -24,10 +24,10 @@ instantiateClean() {
 
 # get latest version
 NEW_VERSION=$(
-  curl -s -H \
+  curl -s -L -H \
     "Accept: application/vnd.github.v3+json" \
     ${GITHUB_TOKEN:+ -H "Authorization: bearer $GITHUB_TOKEN"} \
-    https://api.github.com/repos/returntocorp/semgrep/releases/latest \
+    https://api.github.com/repos/semgrep/semgrep/releases/latest \
   | jq -r '.tag_name'
 )
 # trim v prefix
@@ -58,7 +58,7 @@ fetchPypi rec {
   version = \"$VERSION\";
   format = \"wheel\";
   dist = python;
-  python = \"cp37.cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311\";
+  python = \"cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311\";
   platform = \"$PLATFORM\";
 }
 "
@@ -101,7 +101,7 @@ update_core_platform "aarch64-darwin"
 OLD_PWD=$PWD
 TMPDIR="$(mktemp -d)"
 # shallow clone to check submodule commits, don't actually need the submodules
-git clone https://github.com/returntocorp/semgrep "$TMPDIR/semgrep" --depth 1 --branch "v$NEW_VERSION"
+git clone https://github.com/semgrep/semgrep "$TMPDIR/semgrep" --depth 1 --branch "v$NEW_VERSION"
 
 get_submodule_commit() {
     OLD_PWD=$PWD