Merge branch 'remove_explicit_np_proto_ver' into 'develop'

Remove explicit numpy and proto versions

See merge request !78

.gitlab-ci.yml

@@ -207,6 +207,17 @@ imports:
   script:
     - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_imports.xml $OTBTF_SRC/test/imports_test.py
 
+numpy_gdal_otb:
+  extends: .applications_test_base
+  script:
+    - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_numpy.xml $OTBTF_SRC/test/numpy_test.py
+
+rio:
+  extends: .applications_test_base
+  script:
+    - sudo pip install rasterio
+    - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_rio.xml $OTBTF_SRC/test/rio_test.py
+
 deploy_cpu-dev-testing:
   stage: Update dev image
   extends: .docker_build_base

Dockerfile

@@ -27,8 +27,7 @@ RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/loc
 RUN pip install --no-cache-dir pip --upgrade
 # NumPy version is conflicting with system's gdal dep and may require venv
 ARG NUMPY_SPEC="==1.22.*"
-ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" packaging requests \
  && pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
 
 # ----------------------------------------------------------------------------

RELEASE_NOTES.txt

@@ -12,6 +12,7 @@ Version 4.0.0alpha (4 apr 2023)
 * Tensorflow version: 2.12.0
 * Fixed Tensorflow error "Cannot register 2 metrics with the same name" + new test
 * Faster CI build thanks to bazel remote cache
+* /home/otbuser/.local/bin added to user path
 
 Version 3.4.0 (22 mar 2023)
 ----------------------------------------------------------------

doc/api_distributed.md

@@ -24,7 +24,7 @@ on all GPUs.
 ## Python code
 
 We can start from the codebase of the fully convolutional model example 
-described in the OTBTF [Python API tutorial](#api_tutorial.html).
+described in the OTBTF [Python API tutorial](api_tutorial.html).
 
 ### Dataset
 

doc/deprecated.md

@@ -35,4 +35,4 @@ training, etc. is done using the so-called `tensorflow.Strategy`
 
 !!! Note
 
-    Read our [tutorial](#api_tutorial.html) to know more on working with Keras!
\ No newline at end of file
+    Read our [tutorial](api_tutorial.html) to know more on working with Keras!
\ No newline at end of file

doc/docker_build.md

@@ -49,7 +49,7 @@ be a different branch of OTB, bazel cache will help you to rebuild everything
 except TF, even if the docker cache was purged (after `docker 
 [system|builder] prune`).
 In order to recycle the cache, bazel config and TF git tag should be exactly 
-the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg` 
+the same, any change in *tools/docker/build-env-tf.sh* and `--build-arg` 
 (if related to bazel env, cuda, mkl, xla...) may result in a fresh new build.
 
 Start a cache daemon - here with max 20GB but 10GB should be enough to save 2 

doc/docker_use.md

@@ -104,7 +104,7 @@ Troubleshooting:
 
 If you want to use optimization flags, change GPUs compute capability, etc. 
 you can build your own docker image using the provided dockerfile. 
-See the [docker build documentation](#docker_build.html).
+See the [docker build documentation](docker_build.html).
 
 ## Older images
 

doc/index.md

@@ -16,7 +16,7 @@
 
 
 This remote module of the [Orfeo ToolBox](https://www.orfeo-toolbox.org) 
-provides a generic, multi-purpose deep learning framework, targeting remote 
+provides a generic, multipurpose deep learning framework, targeting remote 
 sensing images processing. It contains a set of new process objects for OTB 
 that internally invoke [Tensorflow](https://www.tensorflow.org/), and new [OTB 
 applications](#otb-applications) to perform deep learning with real-world 
@@ -50,7 +50,7 @@ set of _patches images_ and delivering samples as `tf.dataset` that can be
 used in your favorite TensorFlow pipelines, or convert your patches into 
 TFRecords. The `otbtf.TFRecords` enables you train networks from TFRecords 
 files, which is quite suited for distributed training. Read more in the 
-[tutorial for keras](otbtf/examples/tensorflow_v2x/fcnn/README.md).
+[tutorial for keras](api_tutorial.html).
 
 ## Examples
 

test/imports_test.py

@@ -19,5 +19,16 @@ class ImportsTest(unittest.TestCase):
         self.assertTrue(tensorflow.__version__)
 
 
+    def test_import_all(self):
+        import otbApplication
+        self.assertTrue(otbApplication.Registry_GetAvailableApplications())
+        import tensorflow
+        self.assertTrue(tensorflow.__version__)
+        from osgeo import gdal
+        self.assertTrue(gdal.__version__)
+        import numpy
+        self.assertTrue(numpy.__version__)
+
+
 if __name__ == '__main__':
     unittest.main()

test/numpy_test.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import pytest
+import unittest
+import otbApplication
+from osgeo import gdal
+from test_utils import resolve_paths
+
+FILENAME = resolve_paths('$DATADIR/fake_spot6.jp2')
+
+class NumpyTest(unittest.TestCase):
+
+    def test_gdal_as_nparr(self):
+        gdal_ds = gdal.Open(FILENAME)
+        band = gdal_ds.GetRasterBand(1)
+        arr = band.ReadAsArray()
+        self.assertTrue(arr.shape)
+
+
+    def test_otb_as_nparr(self):
+        app = otbApplication.Registry.CreateApplication('ExtractROI')
+        app.SetParameterString("in", FILENAME)
+        app.Execute()
+        arr = app.GetVectorImageAsNumpyArray('out')
+        self.assertTrue(arr.shape)
+
+    def test_gdal_and_otb_np(self):
+        gdal_ds = gdal.Open(FILENAME)
+        band = gdal_ds.GetRasterBand(1)
+        arr = band.ReadAsArray()
+        app = otbApplication.Registry.CreateApplication('ExtractROI')
+        app.SetImageFromNumpyArray('in', arr)
+        app.SetParameterInt('startx', 0)
+        app.SetParameterInt('starty', 0)
+        app.SetParameterInt('sizex', 10)
+        app.SetParameterInt('sizey', 10)
+        app.Execute()
+        arr2 = app.GetVectorImageAsNumpyArray('out')
+        self.assertTrue(arr2.shape)
+
+if __name__ == '__main__':
+    unittest.main()

test/rio_test.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import pytest
+import unittest
+import rasterio
+import rasterio.features
+import rasterio.warp
+from test_utils import resolve_paths
+
+FILENAME = resolve_paths('$DATADIR/fake_spot6.jp2')
+
+class NumpyTest(unittest.TestCase):
+
+    def test_rio_read_md(self):
+        with rasterio.open(FILENAME) as dataset:
+            # Read the dataset's valid data mask as a ndarray.
+            mask = dataset.dataset_mask()
+
+            # Extract feature shapes and values from the array.
+            for geom, val in rasterio.features.shapes(
+                    mask, transform=dataset.transform
+            ):
+                # Transform shapes from the dataset's own coordinate
+                # reference system to CRS84 (EPSG:4326).
+                geom = rasterio.warp.transform_geom(
+                    dataset.crs, 'EPSG:4326', geom, precision=6
+                )
+                self.assertTrue(geom)
+
+
+    def test_import_all(self):
+        import otbApplication
+        self.assertTrue(otbApplication.Registry_GetAvailableApplications())
+        import tensorflow
+        self.assertTrue(tensorflow.__version__)
+        from osgeo import gdal
+        self.assertTrue(gdal.__version__)
+        import numpy
+        self.assertTrue(numpy.__version__)
+        self.test_rio_read_md()
+        import otbtf
+        self.assertTrue(otbtf.__version__)
+
+
+if __name__ == '__main__':
+    unittest.main()