python310Packages.txtai: init at 5.5.1

pkgs/development/python-modules/txtai/default.nix

@@ -0,0 +1,156 @@
+{
+  lib
+, buildPythonPackage
+, pythonOlder
+, fetchFromGitHub
+, pythonRelaxDepsHook
+# propagated build input
+, faiss
+, torch
+, transformers
+, huggingface-hub
+, numpy
+, pyyaml
+, regex
+# optional-dependencies
+, aiohttp
+, fastapi
+, uvicorn
+# TODO add apache-libcloud
+# , apache-libcloud
+, rich
+, duckdb
+, pillow
+, networkx
+, python-louvain
+, onnx
+, onnxruntime
+, soundfile
+, scipy
+, ttstokenizer
+, beautifulsoup4
+, nltk
+, pandas
+, tika
+, imagehash
+, timm
+, fasttext
+, sentencepiece
+, accelerate
+, onnxmltools
+, annoy
+, hnswlib
+# TODO add pymagnitude-lite
+#, pymagnitude-lite
+, scikit-learn
+, sentence-transformers
+, croniter
+, openpyxl
+, requests
+, xmltodict
+# native check inputs
+, unittestCheckHook
+}:
+let
+  version = "5.5.1";
+  api = [ aiohttp fastapi uvicorn ];
+  # cloud = [ apache-libcloud ];
+  console = [ rich ];
+
+  database = [ duckdb pillow ];
+
+  graph = [ networkx python-louvain ];
+
+  model = [ onnx onnxruntime ];
+
+  pipeline-audio = [ onnx onnxruntime soundfile scipy ttstokenizer ];
+  pipeline-data = [ beautifulsoup4 nltk pandas tika ];
+  pipeline-image = [ imagehash pillow timm ];
+  pipeline-text = [ fasttext sentencepiece ];
+  pipeline-train = [ accelerate onnx onnxmltools onnxruntime ];
+  pipeline = pipeline-audio ++ pipeline-data ++ pipeline-image ++ pipeline-text ++ pipeline-train;
+
+  similarity = [
+    annoy
+    fasttext
+    hnswlib
+    # pymagnitude-lite
+    scikit-learn
+    sentence-transformers
+  ];
+  workflow = [
+    # apache-libcloud
+    croniter
+    openpyxl
+    pandas
+    pillow
+    requests
+    xmltodict
+  ];
+  all = api ++ console ++ database ++ graph ++ model ++ pipeline ++ similarity ++ workflow;
+
+  optional-dependencies = {
+    inherit api console database graph model pipeline-audio pipeline-image
+      pipeline-text pipeline-train pipeline similarity workflow all;
+  };
+in
+buildPythonPackage {
+  pname = "txtai";
+  inherit version;
+  format = "setuptools";
+
+  disable = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "neuml";
+    repo = "txtai";
+    rev = "v${version}";
+    hash = "sha256-h6TwWzLYfFg5x2QMIstAZ5pkxfHobBU+b4gb0HiayzY=";
+  };
+
+  nativeBuildInputs = [
+    pythonRelaxDepsHook
+  ];
+
+  pythonRemoveDeps = [
+    # We call it faiss, not faiss-cpu.
+    "faiss-cpu"
+  ];
+
+  propagatedBuildInputs = [
+    faiss
+    torch
+    transformers
+    huggingface-hub
+    numpy
+    pyyaml
+    regex
+  ];
+
+  passthru.optional-dependencies = optional-dependencies;
+
+  pythonImportsCheck = [ "txtai" ];
+
+  # some tests hang forever
+  doCheck = false;
+
+  preCheck = ''
+    export TRANSFORMERS_CACHE=$(mktemp -d)
+  '';
+
+  nativeCheckInputs = [
+    unittestCheckHook
+  ] ++ optional-dependencies.api ++ optional-dependencies.similarity;
+
+  unittestFlagsArray = [
+    "-s" "test/python" "-v"
+  ];
+
+  meta = with lib; {
+    description = "Semantic search and workflows powered by language models";
+    changelog = "https://github.com/neuml/txtai/releases/tag/v${version}";
+    homepage = "https://github.com/neuml/txtai";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ happysalada ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -12936,6 +12936,8 @@ self: super: with self; {
 
   txrequests = callPackage ../development/python-modules/txrequests { };
 
+  txtai = callPackage ../development/python-modules/txtai { };
+
   txtorcon = callPackage ../development/python-modules/txtorcon { };
 
   txzmq = callPackage ../development/python-modules/txzmq { };