Merge pull request #168057 from samuela/samuela/beam
python3Packages.apache-beam: 2.36.0 -> 2.37.0
This commit is contained in:
Samuel Ainsworth
GitHub
commit
b9514b36b2
3 changed files with 16 additions and 80 deletions
|
|
@ -25,8 +25,6 @@
|
|||
, pydot
|
||||
, pyhamcrest
|
||||
, pymongo
|
||||
, pytest-timeout
|
||||
, pytest-xdist
|
||||
, pytestCheckHook
|
||||
, python
|
||||
, pythonAtLeast
|
||||
|
|
@ -43,26 +41,21 @@
|
|||
|
||||
buildPythonPackage rec {
|
||||
pname = "apache-beam";
|
||||
version = "2.36.0";
|
||||
version = "2.37.0";
|
||||
disabled = pythonAtLeast "3.10";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "apache";
|
||||
repo = "beam";
|
||||
rev = "v${version}";
|
||||
sha256 = "sha256-f+ICbKSwNjkhrTCCZwxbmqZlQ1+dQSTRag1IflWsqYg=";
|
||||
sha256 = "sha256-FmfTxRLqXUHhhAZIxCRx2+phX0bmU5rIHaftBU4yBJY=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
./relax-deps.patch
|
||||
# Fixes https://issues.apache.org/jira/browse/BEAM-9324
|
||||
./fix-cython.patch
|
||||
];
|
||||
|
||||
# See https://github.com/NixOS/nixpkgs/issues/156957.
|
||||
postPatch = ''
|
||||
substituteInPlace setup.py \
|
||||
--replace "typing-extensions>=3.7.0,<4" "typing-extensions" \
|
||||
--replace "dill>=0.3.1.1,<0.3.2" "dill" \
|
||||
--replace "httplib2>=0.8,<0.20.0" "httplib2" \
|
||||
--replace "pyarrow>=0.15.1,<7.0.0" "pyarrow"
|
||||
'';
|
||||
|
||||
|
|
@ -109,8 +102,6 @@ buildPythonPackage rec {
|
|||
parameterized
|
||||
psycopg2
|
||||
pyhamcrest
|
||||
pytest-timeout
|
||||
pytest-xdist
|
||||
pytestCheckHook
|
||||
pyyaml
|
||||
requests-mock
|
||||
|
|
@ -123,6 +114,18 @@ buildPythonPackage rec {
|
|||
preCheck = "cd $out/lib/${python.libPrefix}/site-packages";
|
||||
|
||||
disabledTestPaths = [
|
||||
# Fails with
|
||||
# _______ ERROR collecting apache_beam/io/external/xlang_jdbcio_it_test.py _______
|
||||
# apache_beam/io/external/xlang_jdbcio_it_test.py:80: in <module>
|
||||
# class CrossLanguageJdbcIOTest(unittest.TestCase):
|
||||
# apache_beam/io/external/xlang_jdbcio_it_test.py:99: in CrossLanguageJdbcIOTest
|
||||
# container_init: Callable[[], Union[PostgresContainer, MySqlContainer]],
|
||||
# E NameError: name 'MySqlContainer' is not defined
|
||||
#
|
||||
# Test relies on the testcontainers package, which is not currently (as of
|
||||
# 2022-04-08) available in nixpkgs.
|
||||
"apache_beam/io/external/xlang_jdbcio_it_test.py"
|
||||
|
||||
# These tests depend on the availability of specific servers backends.
|
||||
"apache_beam/runners/portability/flink_runner_test.py"
|
||||
"apache_beam/runners/portability/samza_runner_test.py"
|
||||
|
|
@ -136,12 +139,6 @@ buildPythonPackage rec {
|
|||
# quite elaborate testing infra with containers and multiple
|
||||
# different runners - I don't expect them to help debugging these
|
||||
# when running via our (= custom from their PoV) testing infra.
|
||||
"testBuildListUnpack"
|
||||
"testBuildTupleUnpack"
|
||||
"testBuildTupleUnpackWithCall"
|
||||
"test_convert_bare_types"
|
||||
"test_incomparable_default"
|
||||
"test_pardo_type_inference"
|
||||
"test_with_main_session"
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
diff --git a/apache_beam/runners/worker/operations.py b/apache_beam/runners/worker/operations.py
|
||||
index 3464c5750c..5921c72b90 100644
|
||||
--- a/apache_beam/runners/worker/operations.py
|
||||
+++ b/apache_beam/runners/worker/operations.py
|
||||
@@ -69,18 +69,6 @@ if TYPE_CHECKING:
|
||||
from apache_beam.runners.worker.statesampler import StateSampler
|
||||
from apache_beam.transforms.userstate import TimerSpec
|
||||
|
||||
-# Allow some "pure mode" declarations.
|
||||
-try:
|
||||
- import cython
|
||||
-except ImportError:
|
||||
-
|
||||
- class FakeCython(object):
|
||||
- @staticmethod
|
||||
- def cast(type, value):
|
||||
- return value
|
||||
-
|
||||
- globals()['cython'] = FakeCython()
|
||||
-
|
||||
_globally_windowed_value = GlobalWindows.windowed_value(None)
|
||||
_global_window_type = type(_globally_windowed_value.windows[0])
|
||||
|
||||
@@ -149,7 +137,7 @@ class ConsumerSet(Receiver):
|
||||
# type: (WindowedValue) -> None
|
||||
self.update_counters_start(windowed_value)
|
||||
for consumer in self.consumers:
|
||||
- cython.cast(Operation, consumer).process(windowed_value)
|
||||
+ consumer.process(windowed_value)
|
||||
self.update_counters_finish()
|
||||
|
||||
def try_split(self, fraction_of_remainder):
|
||||
@@ -345,7 +333,7 @@ class Operation(object):
|
||||
|
||||
def output(self, windowed_value, output_index=0):
|
||||
# type: (WindowedValue, int) -> None
|
||||
- cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
|
||||
+ self.receivers[output_index].receive(windowed_value)
|
||||
|
||||
def add_receiver(self, operation, output_index=0):
|
||||
# type: (Operation, int) -> None
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
diff --git a/setup.py b/setup.py
|
||||
index 9429459622..2727b3becb 100644
|
||||
--- a/setup.py
|
||||
+++ b/setup.py
|
||||
@@ -136,12 +136,12 @@ REQUIRED_PACKAGES = [
|
||||
# version of dill. It is best to use the same version of dill on client and
|
||||
# server, therefore list of allowed versions is very narrow.
|
||||
# See: https://github.com/uqfoundation/dill/issues/341.
|
||||
- 'dill>=0.3.1.1,<0.3.2',
|
||||
+ 'dill>=0.3.1.1',
|
||||
'fastavro>=0.21.4,<2',
|
||||
'grpcio>=1.29.0,<2',
|
||||
'hdfs>=2.1.0,<3.0.0',
|
||||
- 'httplib2>=0.8,<0.20.0',
|
||||
- 'numpy>=1.14.3,<1.21.0',
|
||||
+ 'httplib2>=0.8',
|
||||
+ 'numpy>=1.14.3',
|
||||
'pymongo>=3.8.0,<4.0.0',
|
||||
'oauth2client>=2.0.1,<5',
|
||||
'protobuf>=3.12.2,<4',
|
||||
Loading…
Reference in a new issue