diff --git a/.github/workflows/official-docker-images.yml b/.github/workflows/official-docker-images.yml deleted file mode 100644 index b2f58113..00000000 --- a/.github/workflows/official-docker-images.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Build Official Docker Images - -on: - push: - branches: - - master - tags: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - # Fetch all history for all tags and branches - with: - fetch-depth: 0 - - # Build - - name: Build docker images - run: | - docker compose build - - # Test (already been run by pytest workflow, but they don't take long...) - - name: Test with pytest within a docker container - run: | - docker run -v $PWD:/coverage --rm so3g sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov /usr/lib/python3/dist-packages/so3g/ test/" - - - name: Report test coverage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install coveralls - coverage combine - coverage report - coveralls --service=github - - # Dockerize - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_PASSWORD }} - - - name: Build and push official docker image - env: - DOCKERHUB_ORG: "simonsobs" - run: | - export DOCKER_TAG=`git describe --tags --always` - - # Tag all images for upload to the registry - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} - - # Upload to docker registry - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index e4ce3b06..00000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run Tests - -on: - push: - branches: - - master - pull_request: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Build docker images - run: | - docker compose build - - - name: Test with pytest within a docker container - run: | - docker run -v $PWD:/coverage --rm so3g sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov /usr/lib/python3/dist-packages/so3g/ test/" - - - name: Report test coverage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install coveralls - coverage combine - coverage report - coveralls --service=github diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..9efc2b6f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,94 @@ +# In general, we try to run on: +# - The oldest supported python +# - The latest stable python that is the common default on most systems and conda +# - (During transitions) The newly released bleeding edge python + +name: Run Tests + +on: + workflow_dispatch: + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Python-${{ matrix.python }} on ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: "3.9" + arch: Linux-x86_64 + - os: ubuntu-latest + python: "3.11" + arch: Linux-x86_64 + - os: ubuntu-latest + python: "3.12" + arch: Linux-x86_64 + + # MacOS compilation with the conda compilers (clang) will not + # work due to CEREAL polymorphic issues. + # - os: macos-latest + # python: "3.10" + # arch: MacOSX-x86_64 + # - os: macos-latest + # python: "3.12" + # arch: MacOSX-x86_64 + # - os: macos-latest + # python: "3.10" + # arch: MacOSX-arm64 + # - os: macos-latest + # python: "3.12" + # arch: MacOSX-arm64 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Conda Base + run: | + sudo rm -rf /usr/share/miniconda \ + && sudo rm -rf /usr/local/miniconda \ + && curl -SL -o miniforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${{ matrix.arch }}.sh \ + && bash miniforge.sh -b -f -p ~/conda \ + && source ~/conda/etc/profile.d/conda.sh \ + && conda activate base \ + && conda update -n base --yes conda + - name: Check Conda Config + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate base \ + && conda info \ + && conda list \ + && conda config --show-sources \ + && conda config --show + - name: Install Dependencies + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda create --yes -n test python=${{ matrix.python }} \ + && conda activate test \ + && conda install --yes --file conda_dev_requirements.txt \ + && pip install -r test-requirements.txt + - name: Install so3g + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate test \ + && CPU_COUNT=2 python3 -m pip install -v . + - name: Run Tests + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate test \ + && export OMP_NUM_THREADS=2 \ + && pytest ./test diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2c7da150..1a9edd44 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -101,7 +101,7 @@ jobs: CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' - CXXFLAGS='-O3 -fPIC -std=c++14' + CXXFLAGS='-O3 -fPIC -std=c++17' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local SO3G_BUILD_BLAS_LIBRARIES='-L/usr/local/lib -lopenblas -fopenmp -lm -lgfortran' @@ -110,7 +110,7 @@ jobs: CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' - CXXFLAGS='-O3 -fPIC -std=c++14' + CXXFLAGS='-O3 -fPIC -std=c++17' CPATH='/usr/local/include' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cd83c01..fe70cdda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,11 +54,11 @@ include_directories(${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR} ) include_directories(${NUMPY_INCLUDE_DIR}) # -# Define the so3g build target. This is a shared library. +# Define the libso3g build target. This is an internal library that holds all the +# linking dependencies. # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY so3g) -add_library(so3g SHARED - src/main.cxx +add_library(so3g OBJECT src/hkagg.cxx src/Intervals.cxx src/Butterworth.cxx @@ -70,9 +70,6 @@ add_library(so3g SHARED src/array_ops.cxx ) -# We could disable the lib prefix on the output library... but let's not. -#set_target_properties(so3g PROPERTIES PREFIX "") - # Disable boost python auto_ptr warnings target_compile_definitions(so3g PUBLIC BOOST_NO_AUTO_PTR) @@ -124,6 +121,15 @@ add_custom_target(so3g-version add_dependencies(so3g so3g-version) +# Add the Python interface module +Python_add_library(_libso3g MODULE WITH_SOABI src/main.cxx) +set_target_properties(_libso3g PROPERTIES PREFIX "") +target_link_libraries(_libso3g PUBLIC so3g) +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Assume Linux-style ld linker + target_link_options(_libso3g PUBLIC "LINKER:--no-as-needed") +endif() + # Make a list of .py files for the library. file(GLOB MY_PYTHONS "${CMAKE_CURRENT_SOURCE_DIR}/python/*.py") @@ -144,7 +150,7 @@ else() set(INSTALL_DEST ${PYTHON_SITE_PACKAGES}/so3g) endif() -install(TARGETS so3g +install(TARGETS _libso3g DESTINATION ${INSTALL_DEST}) install(FILES ${MY_PYTHONS} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1ec32b50..00000000 --- a/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# so3g -# A containerized so3g installation. - -# Build on spt3g base image -FROM simonsobs/spt3g:0.3-289-g4bd3275 - -# Set locale -ENV LANG C.UTF-8 - -# Build tools needed for pixell; blas needed for so3g. -RUN apt update && apt install -y \ - build-essential \ - automake \ - gfortran \ - libopenblas-openmp-dev \ - libbz2-dev \ - python-is-python3 - -# Set the working directory -WORKDIR /app_lib/so3g - -# Copy the current directory contents into the container -ADD . /app_lib/so3g - -# Install any needed packages specified in requirements.txt -RUN pip3 install -r requirements.txt -RUN pip3 install -r test-requirements.txt - -# Build so3g -RUN /bin/bash /app_lib/so3g/docker/so3g-setup.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 59cfb631..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3.2' -services: - - so3g: - image: "so3g" - build: . diff --git a/docker/qpoint-setup.sh b/docker/qpoint-setup.sh deleted file mode 100644 index a23dfd8e..00000000 --- a/docker/qpoint-setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -QP_VER=1.11.2 -git clone https://github.com/arahlin/qpoint.git --branch $QP_VER -cd qpoint -python3 setup.py install diff --git a/docker/so3g-setup.sh b/docker/so3g-setup.sh deleted file mode 100644 index f8594493..00000000 --- a/docker/so3g-setup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -mkdir -p build -cd build -cmake \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DPython_EXECUTABLE=$(which python3) \ - .. -make -j 2 -make install diff --git a/include/G3SuperTimestream.h b/include/G3SuperTimestream.h index 654c3083..26ec0768 100644 --- a/include/G3SuperTimestream.h +++ b/include/G3SuperTimestream.h @@ -106,7 +106,14 @@ namespace cereal { } G3_POINTERS(G3SuperTimestream); -G3_SERIALIZABLE(G3SuperTimestream, 0); + +CEREAL_CLASS_VERSION(G3SuperTimestream, 0); +CEREAL_REGISTER_TYPE_WITH_NAME(G3SuperTimestream, "G3SuperTimestream"); +CEREAL_REGISTER_POLYMORPHIC_RELATION(G3FrameObject, G3SuperTimestream); + +CEREAL_FORCE_DYNAMIC_INIT(libso3g); + +// G3_SERIALIZABLE(G3SuperTimestream, 0); class g3supertimestream_exception : std::exception { diff --git a/python/__init__.py b/python/__init__.py index 20baae72..0101bc11 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -1,5 +1,4 @@ import os -import numpy as np if os.getenv('DOCS_BUILD') == '1': @@ -9,10 +8,10 @@ from . import spt3g from spt3g import core as spt3g_core - # Our library is called libso3g.{suffix}, but will load into module - # namespace so3g. - from .load_pybindings import load_pybindings - load_pybindings([__path__[0] + '/libso3g'], name='so3g') + # Load all symbols from our compiled extension. This is needed for backwards + # compatibility with external packages that expect those symbols to exist based + # on how the previous loader worked. + from ._libso3g import * # Version is computed by versioneer. __version__ = version() diff --git a/python/hk/cli.py b/python/hk/cli.py index 2cba3291..8257800c 100644 --- a/python/hk/cli.py +++ b/python/hk/cli.py @@ -1,11 +1,11 @@ -import so3g -from spt3g import core import numpy as np import os import sys import csv import argparse +from spt3g import core + _UNITS = { 'bytes': 1, @@ -49,7 +49,7 @@ def get_parser(): # Main "mode" subprocessors. # "list-files" - p = cmdsubp.add_parser( + _ = cmdsubp.add_parser( 'list-files', parents=[data_args, output_args], help="Report per-file stats.", @@ -62,7 +62,7 @@ def get_parser(): """) # "list-provs" - p = cmdsubp.add_parser( + _ = cmdsubp.add_parser( 'list-provs', parents=[data_args, output_args], help="List all data providers (feeds).", @@ -77,7 +77,7 @@ def get_parser(): """) # "list-fields" - p = cmdsubp.add_parser( + _ = cmdsubp.add_parser( 'list-fields', parents=[data_args, output_args], help="List all data field names.", @@ -242,7 +242,6 @@ def main(args=None): if args.mode == 'list-files': rows = [] - file_list = get_file_list for filename in get_file_list(args): file_size = os.path.getsize(filename) clean_exit = True @@ -250,7 +249,7 @@ def main(args=None): while True: try: f = r.Process(None) - except: + except Exception: clean_exit = False break end = r.tell() @@ -318,7 +317,7 @@ def main(args=None): n = len(block.times) for k in keys: field = addr + '.' + k - if not field in counts: + if field not in counts: counts[field] = 0 counts[field] += n header = ['field_name', 'samples'] diff --git a/python/hk/getdata.py b/python/hk/getdata.py index a7de3c34..8ac7fa5b 100644 --- a/python/hk/getdata.py +++ b/python/hk/getdata.py @@ -18,9 +18,10 @@ import numpy as np import datetime as dt - -import so3g from spt3g import core +from .. import _libso3g as libso3g + +from .translator import HKTranslator hk_logger = logging.getLogger(__name__) @@ -92,7 +93,7 @@ def __init__(self, field_groups=None): self.field_groups = list(field_groups) # A translator is used to update frames, on the fly, to the # modern schema assumed here. - self.translator = so3g.hk.HKTranslator() + self.translator = HKTranslator() def _get_groups(self, fields=None, start=None, end=None, short_match=False): @@ -136,7 +137,7 @@ def _get_groups(self, fields=None, start=None, end=None, in any group. """ - span = so3g.IntervalsDouble() + span = libso3g.IntervalsDouble() if start is None: start = span.domain[0] if end is None: @@ -170,7 +171,7 @@ def _get_groups(self, fields=None, start=None, end=None, continue elif full_field not in fields: continue - if not key_field in field_map: + if key_field not in field_map: field_map[key_field] = [fg] else: field_map[key_field].append(fg) @@ -292,7 +293,7 @@ def check_overlap(time_range): for time_range, n, filename, byte_offset, block_index in all_frame_refs: if not check_overlap(time_range): continue - if not filename in files: + if filename not in files: files[filename] = {} if byte_offset not in files[filename]: files[filename][byte_offset] = [] @@ -484,7 +485,7 @@ def __init__(self, pre_proc_dir=None, pre_proc_mode=None): self.field_groups = [] self.frame_info = [] self.counter = -1 - self.translator = so3g.hk.HKTranslator() + self.translator = HKTranslator() self.pre_proc_dir = pre_proc_dir self.pre_proc_mode = pre_proc_mode @@ -516,7 +517,7 @@ def Process(self, f, index_info=None): vers = f.get('hkagg_version', 0) assert(vers == 2) - if f['hkagg_type'] == so3g.HKFrameType.session: + if f['hkagg_type'] == libso3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: @@ -526,7 +527,7 @@ def Process(self, f, index_info=None): (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id - elif f['hkagg_type'] == so3g.HKFrameType.status: + elif f['hkagg_type'] == libso3g.HKFrameType.status: # If a provider has disappeared, flush its information into a # FieldGroup. prov_cands = [_HKProvider.from_g3(p) for p in f['providers']] @@ -539,10 +540,9 @@ def Process(self, f, index_info=None): for prov_id in to_flush: self.flush([prov_id]) - elif f['hkagg_type'] == so3g.HKFrameType.data: + elif f['hkagg_type'] == libso3g.HKFrameType.data: # Data frame -- merge info for this provider. prov = self.providers[f['prov_id']] - representatives = prov.blocks.keys() for bidx, (bname, b) in enumerate(zip(f['block_names'], f['blocks'])): assert(isinstance(b, core.G3TimesampleMap)) @@ -560,7 +560,7 @@ def Process(self, f, index_info=None): 'count': len(b.times)} ii.update(index_info) prov.blocks[bname]['index_info'].append(ii) - + else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') @@ -665,7 +665,7 @@ def process_file_with_cache(self, filename): with open(path, 'wb') as pkfl: pickle.dump(hksc, pkfl) if self.pre_proc_mode is not None: - os.chmod( path, self.pre_proc_mode ) + os.chmod( path, self.pre_proc_mode ) self.field_groups += hksc.field_groups self.counter += hksc.counter @@ -711,17 +711,17 @@ class _FieldGroup: def __init__(self, prefix, fields, start, end, index_info): self.prefix = prefix self.fields = list(fields) - self.cover = so3g.IntervalsDouble().add_interval(start, end) + self.cover = libso3g.IntervalsDouble().add_interval(start, end) self.index_info = index_info def __repr__(self): try: return '_FieldGroup("%s", %i fields, %i segments)' % ( self.prefix, len(self.fields), len(self.index_info)) - except: + except Exception: return '_FieldGroup()' -def to_timestamp(some_time, str_format=None): +def to_timestamp(some_time, str_format=None): """Convert the argument to a unix timestamp. Args: @@ -736,25 +736,25 @@ def to_timestamp(some_time, str_format=None): float: Unix timestamp corresponding to some_time. """ - - if type(some_time) == dt.datetime: + + if isinstance(some_time, dt.datetime): return some_time.astimezone(dt.timezone.utc).timestamp() - if type(some_time) == int or type(some_time) == float: + if isinstance(some_time, (int, float)): return some_time - if type(some_time) == str: + if isinstance(some_time, str): if str_format is not None: return to_timestamp(pytz.utc.localize( dt.datetime.strptime(some_time, str_format ))) str_options = ['%Y-%m-%d', '%Y-%m-%d %H:%M', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f'] for option in str_options: try: return to_timestamp(pytz.utc.localize( dt.datetime.strptime(some_time, option ))) - except: + except Exception: continue raise ValueError('Could not process string into date object, options are: {}'.format(str_options)) - + raise ValueError('Type of date / time indication is invalid, accepts datetime, int, float, and string') -def load_range(start, stop, fields=None, alias=None, +def load_range(start, stop, fields=None, alias=None, data_dir=None, config=None, pre_proc_dir=None, pre_proc_mode=None, folder_patterns=None, strict=True): """Args: @@ -827,7 +827,7 @@ def load_range(start, stop, fields=None, alias=None, hk_logger.warning('''load_range has a config file - data_dir, fields, and alias are ignored''') with open(config, 'r') as f: setup = yaml.load(f, Loader=yaml.FullLoader) - + if 'data_dir' not in setup.keys(): raise ValueError('load_range config file requires data_dir entry') data_dir = setup['data_dir'] @@ -838,14 +838,14 @@ def load_range(start, stop, fields=None, alias=None, for k in setup['field_list']: fields.append( setup['field_list'][k]) alias.append( k ) - + if data_dir is None and 'OCS_DATA_DIR' not in os.environ.keys(): raise ValueError('if $OCS_DATA_DIR is not defined a data directory must be passed to getdata') if data_dir is None: data_dir = os.environ['OCS_DATA_DIR'] hk_logger.debug('Loading data from {}'.format(data_dir)) - + start_ctime = to_timestamp(start) - 3600 stop_ctime = to_timestamp(stop) + 3600 @@ -877,20 +877,20 @@ def load_range(start, stop, fields=None, alias=None, continue try: t = int(file[:-3]) - except: + except Exception: hk_logger.debug('{} does not have the right format, skipping'.format(file)) continue if t >= start_ctime-3600 and t <=stop_ctime+3600: hk_logger.debug('Processing {}'.format(base+'/'+file)) hksc.process_file_with_cache( base+'/'+file) - + cat = hksc.finalize() start_ctime = to_timestamp(start) stop_ctime = to_timestamp(stop) - + all_fields,_ = cat.get_fields() - + if fields is None: fields = all_fields if alias is not None: @@ -898,7 +898,7 @@ def load_range(start, stop, fields=None, alias=None, hk_logger.error('if provided, alias needs to be the length of fields') else: alias = fields - + # Single pass load. keepers = [] for name, field in zip(alias, fields): @@ -969,7 +969,7 @@ def load_range(start, stop, fields=None, alias=None, # This is the easy way, which just gives you one timeline per # requested field. x1, y1 = cat.simple(field_name) - + assert np.all(np.array(x0) == x1) and np.all(np.array(y0) == y1) import pylab as pl diff --git a/python/hk/scanner.py b/python/hk/scanner.py index 9fabbfc1..b7061c17 100644 --- a/python/hk/scanner.py +++ b/python/hk/scanner.py @@ -1,12 +1,14 @@ -import so3g -from spt3g import core import numpy as np -from so3g import hk +from spt3g import core + +from .. import _libso3g as libso3g +from .translator import HKTranslator + class HKScanner: """Module that scans and reports on HK archive contents and compliance. - + Attributes: stats (dict): A nested dictionary of statistics that are updated as frames are processed by the module. Elements: @@ -62,7 +64,7 @@ def __call__(self, f): vers = f.get('hkagg_version', 0) self.stats['versions'][vers] = self.stats['versions'].get(vers, 0) + 1 - if f['hkagg_type'] == so3g.HKFrameType.session: + if f['hkagg_type'] == libso3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: @@ -73,13 +75,13 @@ def __call__(self, f): self.session_id = session_id self.stats['n_session'] += 1 - elif f['hkagg_type'] == so3g.HKFrameType.status: + elif f['hkagg_type'] == libso3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False - + # New providers? for p in now_prov_id: info = self.providers.get(p) @@ -102,7 +104,7 @@ def __call__(self, f): 'block_streams_map': {}, # Map from field name to block name. } - elif f['hkagg_type'] == so3g.HKFrameType.data: + elif f['hkagg_type'] == libso3g.HKFrameType.data: info = self.providers[f['prov_id']] vers = f.get('hkagg_version', 0) @@ -178,7 +180,7 @@ def __call__(self, f): 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 - + else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') @@ -203,6 +205,6 @@ def __call__(self, f): p = core.G3Pipeline() p.Add(core.G3Reader(f)) if args.translate: - p.Add(hk.HKTranslator(target_version=args.target_version)) + p.Add(HKTranslator(target_version=args.target_version)) p.Add(HKScanner()) p.Run() diff --git a/python/hk/session.py b/python/hk/session.py index f5b79986..43820305 100644 --- a/python/hk/session.py +++ b/python/hk/session.py @@ -1,9 +1,10 @@ -import so3g -from spt3g import core import time import os import binascii +from spt3g import core +from .. import _libso3g as libso3g + class HKSessionHelper: def __init__(self, session_id=None, start_time=None, hkagg_version=None, @@ -93,7 +94,7 @@ def session_frame(self): """ f = core.G3Frame() f.type = core.G3FrameType.Housekeeping - f['hkagg_type'] = so3g.HKFrameType.session + f['hkagg_type'] = libso3g.HKFrameType.session f['hkagg_version'] = self.hkagg_version f['session_id'] = self.session_id f['start_time'] = self.start_time @@ -110,7 +111,7 @@ def status_frame(self, timestamp=None): timestamp = time.time() f = core.G3Frame() f.type = core.G3FrameType.Housekeeping - f['hkagg_type'] = so3g.HKFrameType.status + f['hkagg_type'] = libso3g.HKFrameType.status f['hkagg_version'] = self.hkagg_version f['session_id'] = self.session_id f['timestamp'] = timestamp @@ -136,7 +137,7 @@ def data_frame(self, prov_id, timestamp=None): f = core.G3Frame() f.type = core.G3FrameType.Housekeeping f['hkagg_version'] = self.hkagg_version - f['hkagg_type'] = so3g.HKFrameType.data + f['hkagg_type'] = libso3g.HKFrameType.data f['session_id'] = self.session_id f['prov_id'] = prov_id f['timestamp'] = timestamp diff --git a/python/hk/translator.py b/python/hk/translator.py index d6c10741..8f962ca3 100644 --- a/python/hk/translator.py +++ b/python/hk/translator.py @@ -1,10 +1,10 @@ """Backwards compatibility for older SO HK schemas. """ - -import so3g -import so3g.hk from spt3g import core +from .. import _libso3g as libso3g + +from .util import get_g3_time class HKTranslator: @@ -89,7 +89,7 @@ def Process(self, f): f['hkagg_version'] = self.target_version # No difference in Session/Status for v0, v1, v2. - if f.get('hkagg_type') != so3g.HKFrameType.data: + if f.get('hkagg_type') != libso3g.HKFrameType.data: return [f] if self.target_version == 0: @@ -103,7 +103,7 @@ def Process(self, f): # Now process the data blocks. for block in orig_blocks: new_block = core.G3TimesampleMap() - new_block.times = so3g.hk.util.get_g3_time(block.t) + new_block.times = get_g3_time(block.t) for k in block.data.keys(): v = block.data[k] new_block[k] = core.G3VectorDouble(v) @@ -122,7 +122,6 @@ def Process(self, f): field_names = list(sorted(block.keys())) block_names.append('block_for_%s' % field_names[0]) assert(len(block_names[-1]) < 256) # What have you done. - orig_block_names = [] f['block_names'] = core.G3VectorString(block_names) return [f] diff --git a/python/hk/tree.py b/python/hk/tree.py index 643519bb..70c10efe 100644 --- a/python/hk/tree.py +++ b/python/hk/tree.py @@ -2,13 +2,13 @@ of attributes. """ - -from so3g.hk import getdata import time import os import yaml import logging +from .getdata import to_timestamp, HKArchiveScanner + logger = logging.getLogger(__name__) @@ -137,11 +137,11 @@ def __init__(self, start=None, stop=None, config=None, if start is None: start = now - 86400 else: - start = getdata.to_timestamp(start) + start = to_timestamp(start) if stop is None: stop = start + 86400 else: - stop = getdata.to_timestamp(stop) + stop = to_timestamp(stop) if aliases is None: aliases = {} @@ -170,7 +170,7 @@ def __init__(self, start=None, stop=None, config=None, # Walk the files -- same approach as load_ranges logger.debug('Scanning %s (pre_proc=%s)' % (data_dir, pre_proc_dir)) - hksc = getdata.HKArchiveScanner(pre_proc_dir=pre_proc_dir) + hksc = HKArchiveScanner(pre_proc_dir=pre_proc_dir) for folder in range(int(start / 1e5), int(stop / 1e5) + 1): base = os.path.join(data_dir, str(folder)) logger.debug(f' ... checking {base}') diff --git a/python/load_pybindings.py b/python/load_pybindings.py deleted file mode 100644 index 9a8ea4d4..00000000 --- a/python/load_pybindings.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Based on spt3g.core.load_bindings. -# -import platform, sys, os - -# Starting in spt3g 0.3-240-ga9d32d5, dload may be used. -from spt3g import dload - - -if platform.system().startswith('freebsd') or platform.system().startswith('FreeBSD'): - # C++ modules are extremely fragile when loaded with RTLD_LOCAL, - # which is what Python uses on FreeBSD by default, and maybe other - # systems. Convince it to use RTLD_GLOBAL. - - # See thread by Abrahams et al: - # http://mail.python.org/pipermail/python-dev/2002-May/024074.html - sys.setdlopenflags(0x102) - -def load_pybindings(paths, name=None, lib_suffix=None): - """ - Load all non-private items from the libraries in the list "paths". - Provide the full path to each library, but without extension. The - .so or .dylib will be appended depending on the system - architecture. The namespace into which the items are imported - will be determined from the first path, unless name= is explicitly - provided. - """ - if lib_suffix is None: - if platform.system().startswith('Darwin'): - # OSX compatibility requires .dylib suffix - lib_suffix = ".dylib" - else: - lib_suffix = ".so" - for path in paths: - if name is None: - name = os.path.split(path)[1] - # Save copy of current module def - mod = sys.modules[name] - m = dload.load_dynamic(name, name, path + lib_suffix) - sys.modules[name] = mod # Don't override Python mod with C++ - - for (k,v) in m.__dict__.items(): - if not k.startswith("_"): - mod.__dict__[k] = v diff --git a/python/proj/__init__.py b/python/proj/__init__.py index d0a9790c..f43301d7 100644 --- a/python/proj/__init__.py +++ b/python/proj/__init__.py @@ -1,11 +1,8 @@ -import so3g -from spt3g import core - from . import quat from . import util from . import mapthreads -from .wcs import Projectionist, ProjectionistHealpix, Ranges, RangesMatrix +from .wcs import Projectionist, ProjectionistHealpix, RangesMatrix from .coords import CelestialSightLine, EarthlySite, Assembly, FocalPlane, SITES from .weather import Weather, weather_factory from .ranges import Ranges, RangesMatrix diff --git a/python/proj/coords.py b/python/proj/coords.py index 54b5d7d5..990e03a3 100644 --- a/python/proj/coords.py +++ b/python/proj/coords.py @@ -1,10 +1,9 @@ -import so3g +import numpy as np + +from .. import _libso3g as libso3g from . import quat from .weather import weather_factory -from collections import OrderedDict - -import numpy as np DEG = np.pi / 180. @@ -224,7 +223,7 @@ def coords(self, fplane=None, output=None): be [n_det,n_samp,{lon,lat,cos2psi,sin2psi}] """ # Get a projector, in CAR. - p = so3g.ProjEng_CAR_TQU_NonTiled((1, 1, 1., 1., 1., 1.)) + p = libso3g.ProjEng_CAR_TQU_NonTiled((1, 1, 1., 1., 1., 1.)) # Pre-process the offsets collapse = (fplane is None) if collapse: @@ -277,7 +276,8 @@ def __init__(self, quats=None, resps=None, dets=None): # quats will be an quat coeff array-2 and resps will be a numpy # array with the right shape, so we don't need to check # for this when we use FocalPlane later - if quats is None: quats = [] + if quats is None: + quats = [] # Asarray needed because G3VectorQuat doesn't handle list of lists, # which we want to be able to accept self.quats = quat.G3VectorQuat(np.asarray(quats)) @@ -370,7 +370,8 @@ def from_xieta(cls, *args, **kwargs): xi, eta, gamma, T, P, Q, U, hwp, dets = cls._xieta_compat(*args, **kwargs) gamma = gamma + np.arctan2(U,Q)/2 P = P * (Q**2+U**2)**0.5 - if hwp: gamma = -gamma + if hwp: + gamma = -gamma # Broadcast everything to 1d xi, eta, gamma, T, P, _ = np.broadcast_arrays(xi, eta, gamma, T, P, [0]) quats = quat.rotation_xieta(xi, eta, gamma) @@ -395,7 +396,8 @@ def __getitem__(self, sel): in which case an ``spt3g.core.quat`` is returned for that detector. This is provided for backwards compatibility.""" # FIXME: old sotodlib compat - remove later - if isinstance(sel, str): return self.quats[self._dets.index(sel)] + if isinstance(sel, str): + return self.quats[self._dets.index(sel)] # We go via .coeffs() here to get around G3VectorQuat's lack # of boolean mask support return FocalPlane(quats=self.coeffs()[sel], resps=self.resps[sel]) @@ -424,7 +426,7 @@ def __setitem__(self, name, q): # expects to be able to build up a focalplane by assigning # quats one at a time if name in self._detmap: - i = self._detmap[i] + i = self._detmap[name] self.quats[i] = q else: self._dets.append(name) diff --git a/python/proj/mapthreads.py b/python/proj/mapthreads.py index 92e71f2e..d7459246 100644 --- a/python/proj/mapthreads.py +++ b/python/proj/mapthreads.py @@ -5,10 +5,17 @@ parallelization. """ - -import so3g import numpy as np +from .. import _libso3g as libso3g + +from . import quat +from .coords import Assembly, FocalPlane +from .wcs import Projectionist + +DEG = np.pi/180. + + def get_num_threads(n_threads=None): """Utility function for computing n_threads. If n_threads is not None, it is returned directly. But if it is None, then the OpenMP @@ -16,7 +23,7 @@ def get_num_threads(n_threads=None): """ if n_threads is None: - return so3g.useful_info()['omp_num_threads'] + return libso3g.useful_info()['omp_num_threads'] return n_threads def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, @@ -77,11 +84,11 @@ def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, active_tiles = [0] # The full assembly, for later. - asm_full = so3g.proj.Assembly.attach(sight, fplane) + asm_full = Assembly.attach(sight, fplane) # Get a Projectionist -- note it can be used with full or # representative assembly. - pmat = so3g.proj.wcs.Projectionist.for_tiled( + pmat = Projectionist.for_tiled( shape, wcs, tile_shape=tile_shape, active_tiles=active_tiles ) if active_tiles is None: @@ -93,9 +100,9 @@ def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, # For the scan direction map, use the "representative" subset # detectors, with polarization direction aligned parallel to # elevation. - xi, eta, gamma = so3g.proj.quat.decompose_xieta(fplane_rep.quats) - fplane_xl = so3g.proj.FocalPlane.from_xieta(xi, eta, gamma*0+90*so3g.proj.DEG) - asm_rep = so3g.proj.Assembly.attach(sight, fplane_xl) + xi, eta, gamma = quat.decompose_xieta(fplane_rep.quats) + fplane_xl = FocalPlane.from_xieta(xi, eta, gamma*0+90*DEG) + asm_rep = Assembly.attach(sight, fplane_xl) sig = np.ones((fplane_xl.ndet, len(asm_rep.Q)), dtype='float32') scan_maps = pmat.to_map(sig, asm_rep, comps='TQU') @@ -107,7 +114,7 @@ def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, phi = np.arctan2(U, Q) / 2 if plot_prefix: - text = 'Qf=%.2f Uf=%.2f phi=%.1f deg' % (Q/T, U/T, phi / so3g.proj.DEG) + text = 'Qf=%.2f Uf=%.2f phi=%.1f deg' % (Q/T, U/T, phi / DEG) for label, _m in tile_iter(scan_maps): for i in range(3): pl.imshow(_m[i], origin='lower') diff --git a/python/proj/quat.py b/python/proj/quat.py index c092d732..3bbd124e 100644 --- a/python/proj/quat.py +++ b/python/proj/quat.py @@ -1,10 +1,7 @@ import numpy as np -try: - from spt3g.core import quat, G3VectorQuat -except ImportError: - # Pre-Oct 2019 versions. - from spt3g.coordinateutils import quat, G3VectorQuat +from spt3g.core import Quat, G3VectorQuat + """We are using the spt3g quaternion containers, i.e. cu3g.G3VectorQuat and cu3g.quat. One way these are nice is that @@ -12,28 +9,25 @@ The component ordering is (1,i,j,k). We are restricted to 0d or 1d, and that's fine.""" -DEG = np.pi/180 - - def euler(axis, angle): """ The quaternion representing of an Euler rotation. - + For example, if axis=2 the computed quaternion(s) will have components: q = (cos(angle/2), 0, 0, sin(angle/2)) - + Parameters ---------- axis : {0, 1, 2} The index of the cartesian axis of the rotation (x, y, z). angle : float or 1-d float array Angle of rotation, in radians. - + Returns ------- - quat or G3VectorQuat, depending on ndim(angle). + Quat or G3VectorQuat, depending on ndim(angle). """ # Either angle or axis or both can be vectors. angle = np.asarray(angle) @@ -43,13 +37,13 @@ def euler(axis, angle): q[..., 0] = c q[..., axis+1] = s if len(shape) == 1: - return quat(*q) + return Quat(*q) return G3VectorQuat(q) def rotation_iso(theta, phi, psi=None): """Returns the quaternion that composes the Euler rotations: - + Qz(phi) Qy(theta) Qz(psi) Note arguments are in radians. @@ -62,9 +56,9 @@ def rotation_iso(theta, phi, psi=None): def rotation_lonlat(lon, lat, psi=0.): """Returns the quaternion that composes the Euler rotations: - + Qz(lon) Qy(pi/2 - lat) Qz(psi) - + Note arguments are in radians. """ return rotation_iso(np.pi/2 - lat, lon, psi) @@ -92,21 +86,21 @@ def rotation_xieta(xi, eta, gamma=0): def decompose_iso(q): """Decomposes the rotation encoded by q into the product of Euler rotations: - + q = Qz(phi) Qy(theta) Qz(psi) - + Parameters ---------- - q : quat or G3VectorQuat + q : Quat or G3VectorQuat The quaternion(s) to be decomposed. - + Returns ------- (theta, phi, psi) : tuple of floats or of 1-d arrays The rotation angles, in radians. """ - if isinstance(q, quat): + if isinstance(q, Quat): a,b,c,d = q.a, q.b, q.c, q.d else: a,b,c,d = np.transpose(q) diff --git a/python/proj/ranges.py b/python/proj/ranges.py index a5fde7f2..5bbdeee2 100644 --- a/python/proj/ranges.py +++ b/python/proj/ranges.py @@ -1,11 +1,13 @@ -import so3g import numpy as np +from .. import _libso3g as libso3g + + """Objects will self report as being of type "RangesInt32" rather than Ranges. But let's try to use so3g.proj.Ranges when testing types and making new ones and stuff.""" -Ranges = so3g.RangesInt32 +Ranges = libso3g.RangesInt32 class RangesMatrix(): @@ -43,7 +45,7 @@ def copy(self): def zeros_like(self): return RangesMatrix([x.zeros_like() for x in self.ranges], child_shape=self.shape[1:]) - + def ones_like(self): return RangesMatrix([x.ones_like() for x in self.ranges], child_shape=self.shape[1:]) @@ -52,7 +54,7 @@ def buffer(self, buff): [x.buffer(buff) for x in self.ranges] ## just to make this work like Ranges.buffer() return self - + def buffered(self, buff): out = self.copy() [x.buffer(buff) for x in out.ranges] @@ -127,7 +129,7 @@ def __add__(self, x): elif self.shape[0] == x.shape[0]: return self.__class__([r + d for r, d in zip(self.ranges, x)]) return self.__class__([r + x for r in self.ranges]) - + def __mul__(self, x): if isinstance(x, Ranges): return self.__class__([d * x for d in self.ranges]) diff --git a/python/proj/util.py b/python/proj/util.py index b9f8f2dc..656db295 100644 --- a/python/proj/util.py +++ b/python/proj/util.py @@ -1,5 +1,3 @@ -import numpy as np - def ces(el, az0, throw, v_scan, t): """Generate a CES scan pattern. diff --git a/python/proj/wcs.py b/python/proj/wcs.py index 8c2005d3..c51c14c3 100644 --- a/python/proj/wcs.py +++ b/python/proj/wcs.py @@ -1,11 +1,15 @@ -import so3g -from . import quat - import numpy as np -from .ranges import Ranges, RangesMatrix +from .. import _libso3g as libso3g + +from . import quat + +from .ranges import RangesMatrix from . import mapthreads +DEG = np.pi / 180. + + # For coordinate systems we use the following abbreviations: # # - DC: Detector coordinates @@ -111,7 +115,8 @@ def get_ProjEng(self, comps='TQU', proj_name=None, get=True, configured geometry. """ - if proj_name is None: proj_name = self.proj_name + if proj_name is None: + proj_name = self.proj_name tile_suffix = 'Tiled' if self.tiling else 'NonTiled' # Interpolation mode @@ -128,7 +133,7 @@ def get_ProjEng(self, comps='TQU', proj_name=None, get=True, if not get: return projeng_name try: - projeng_cls = getattr(so3g, projeng_name) + projeng_cls = getattr(libso3g, projeng_name) except AttributeError: raise ValueError(f'There is no projector implemented for ' f'pixelization "{proj_name}", components ' @@ -445,7 +450,7 @@ def get_active_tiles(self, assembly, assign=False): tiles = np.nonzero(hits)[0] hits = hits[tiles] if assign is True: - assign = so3g.useful_info()['omp_num_threads'] + assign = libso3g.useful_info()['omp_num_threads'] if assign > 0: group_n = np.array([0 for g in range(assign)]) group_tiles = [[] for _ in group_n] @@ -454,7 +459,7 @@ def get_active_tiles(self, assembly, assign=False): idx = group_n.argmin() group_n[idx] += _n group_tiles[idx].append(_t) - imax = np.argmax(group_n) + # imax = np.argmax(group_n) # max_ratio = group_n[imax] / np.mean(np.concatenate([group_n[:imax], group_n[imax+1:]])) # if len(group_n)>1 and max_ratio > 1.1: # print(f"Warning: Threads poorly balanced. Max/mean hits = {max_ratio}") @@ -534,17 +539,17 @@ def get_q(wcs): # This is typical for cylindrical projections. assert((delta0 >= 0 and wcs.wcs.lonpole == 180.0) or (delta0 <= 0 and wcs.wcs.lonpole == 0.0)) - Q = (quat.euler(1, delta0 * quat.DEG) * - quat.euler(2, -alpha0 * quat.DEG)) + Q = (quat.euler(1, delta0 * DEG) * + quat.euler(2, -alpha0 * DEG)) elif (wcs.wcs.phi0 == 0. and wcs.wcs.theta0 == 90.): # This is typical for zenithal projections. assert(wcs.wcs.lonpole == 180.0) Q = (quat.euler(2, np.pi) * - quat.euler(1, (delta0 - 90)*quat.DEG) * - quat.euler(2, -alpha0 * quat.DEG)) + quat.euler(1, (delta0 - 90)*DEG) * + quat.euler(2, -alpha0 * DEG)) else: - raise ValueError(f'Unimplemented NSC reference (phi0,theta0)=' - '({wcs.wcs.phi0:.2f},{wcs.wcs.theta0:.2f})') + raise ValueError('Unimplemented NSC reference (phi0,theta0)=' + f'({wcs.wcs.phi0:.2f},{wcs.wcs.theta0:.2f})') return Q def __init__(self): @@ -595,7 +600,7 @@ def for_geom(cls, shape, wcs, interpol=None): self.q_celestial_to_native = self.get_q(wcs) # Store the grid info. - self.cdelt = np.array(wcs.wcs.cdelt) * quat.DEG + self.cdelt = np.array(wcs.wcs.cdelt) * DEG self.crpix = np.array(wcs.wcs.crpix) # Pixel interpolation mode @@ -634,8 +639,8 @@ def for_source_at(cls, alpha0, delta0, gamma0=0., assert(gamma0 == 0.) self.q_celestial_to_native = ( quat.euler(2, np.pi) - * quat.euler(1, (delta0 - 90)*quat.DEG) - * quat.euler(2, -alpha0 * quat.DEG)) + * quat.euler(1, (delta0 - 90)*DEG) + * quat.euler(2, -alpha0 * DEG)) return self @classmethod @@ -720,7 +725,7 @@ def __init__(self): self._q_fp_to_celestial = None self.active_tiles = None self.proj_name = None - self.q_celestial_to_native = quat.quat(1,0,0,0) + self.q_celestial_to_native = quat.Quat(1,0,0,0) self.interpol = 'nearest' self.tiling = None @@ -771,7 +776,7 @@ def compute_nside_tile(self, assembly, nActivePerThread=5, nThreads=None): nActive = len(self.get_active_tiles(assembly)['active_tiles']) fsky = nActive / (12 * nside_tile0**2) if nThreads is None: - nThreads = so3g.useful_info()['omp_num_threads'] + nThreads = libso3g.useful_info()['omp_num_threads'] # nside_tile is smallest power of 2 satisfying nTile >= nActivePerThread * nthread / fsky self.nside_tile = int(2**np.ceil(0.5 * np.log2(nActivePerThread * nThreads / (12 * fsky)))) self.nside_tile = min(self.nside_tile, self.nside) diff --git a/python/smurf/reader.py b/python/smurf/reader.py index 1565bbe0..0f0a48ac 100644 --- a/python/smurf/reader.py +++ b/python/smurf/reader.py @@ -1,12 +1,11 @@ -import so3g -from spt3g import core import numpy as np import pickle -import datetime, time -import sys, os import warnings import argparse +from spt3g import core + + def g3_to_array(g3file, verbose=False): """ Takes a G3 file output from the SMuRF archiver and reads to a numpy array. @@ -22,7 +21,7 @@ def g3_to_array(g3file, verbose=False): data : array of arrays, where each internal array is a SMuRF channel """ frames = [fr for fr in core.G3File(g3file)] - + data=[] frametimes = [] @@ -35,13 +34,13 @@ def g3_to_array(g3file, verbose=False): warnings.warn('Wrong frame type') strtimes = np.hstack(frametimes) - + times = [] for strt in strtimes: t=core.G3Time(strt).time/core.G3Units.s times.append(t) times = np.asarray(times) - + channums = [] i=0 diff --git a/python/smurf/smurf_archive.py b/python/smurf/smurf_archive.py index 6605f58f..969aeaeb 100644 --- a/python/smurf/smurf_archive.py +++ b/python/smurf/smurf_archive.py @@ -1,18 +1,19 @@ +import ast +from collections import namedtuple +import datetime as dt +from enum import Enum +import os + import sqlalchemy as db from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker, relationship, backref +from sqlalchemy.orm import sessionmaker, relationship -from spt3g import core -import so3g -import datetime as dt -import os -from tqdm import tqdm import numpy as np +from tqdm import tqdm import yaml -import ast -from collections import namedtuple -from enum import Enum + +from spt3g import core Base = declarative_base() @@ -164,7 +165,7 @@ def add_file(self, path, session): db_file = Files(path=path) session.add(db_file) - reader = so3g.G3Reader(path) + reader = core.G3Reader(path) total_channels = 0 file_start, file_stop = None, None @@ -326,7 +327,7 @@ def load_data(self, start, end, show_pb=True, load_biases=True): for frame_info in tqdm(frames, total=num_frames, disable=(not show_pb)): file = frame_info.file.path if file != cur_file: - reader = so3g.G3Reader(file) + reader = core.G3Reader(file) cur_file = file reader.seek(frame_info.offset) @@ -400,7 +401,7 @@ def load_status(self, time, show_pb=False): for frame_info in tqdm(status_frames.all(), disable=(not show_pb)): file = frame_info.file.path if file != cur_file: - reader = so3g.G3Reader(file) + reader = core.G3Reader(file) cur_file = file reader.seek(frame_info.offset) frame = reader.Process(None)[0] diff --git a/setup.py b/setup.py index 8bb4a75a..db60a147 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ import os import sys -import sysconfig import re import subprocess as sp import glob @@ -19,26 +18,9 @@ # Absolute path to the directory with this file topdir = Path(__file__).resolve().parent -# The version of spt3g we will be installing. Get this from the -# Dockerfile for consistency. +# The version of spt3g we will be installing. def get_spt3g_version(): - dockerfile = os.path.join(topdir, "Dockerfile") - ver = None - linepat = re.compile(r".*simonsobs/spt3g:(.*)\s*") - verpat = re.compile(r".*-g(.*)") - with open(dockerfile, "r") as f: - for line in f: - mat = linepat.match(line) - if mat is not None: - fullver = mat.group(1) - vermat = verpat.match(fullver) - if vermat is None: - # This must be an actual tag - ver = fullver - else: - # Extract the short hash - ver = vermat.group(1) - return ver + return "66f373d1b4bf9076fe0af8236eed022c3d006664" upstream_spt3g_version = get_spt3g_version() @@ -56,7 +38,7 @@ def get_version(): ver_info = ver_function() ver = ver_info["version"] sys.path.pop(0) - except: + except Exception: raise RuntimeError("Cannot call get_versions() from version_h.py!") return ver @@ -128,7 +110,7 @@ def build_common(src_dir, build_dir, install_dir, cmake_extra, debug, pkg, versi cxxflags = f"{cxxflags} -DVERSION_INFO='{version}'" if sys.platform.lower() == "darwin": cmake_args += ["-DCMAKE_SHARED_LINKER_FLAGS='-undefined dynamic_lookup'"] - + # Add numpy includes numpy_inc = np.get_include() cxxflags += f" -I{numpy_inc}" @@ -206,7 +188,7 @@ def run(self): Perform build_cmake before doing the 'normal' stuff """ for extension in self.extensions: - if extension.name == "so3g.libso3g": + if extension.name == "so3g._libso3g": # We just trigger this on one of the extensions. build_cmake() # will actually build everything. self.build_cmake() @@ -234,13 +216,9 @@ def build_cmake(self): # CMake build directory for spt3g temp_spt3g = os.path.join(temp_build, "spt3g") - # The python module in the spt3g build directory. This contains - # the compiled libraries and symlinks to the python source. - spt3g_python_dir = os.path.join(temp_spt3g, "spt3g") - # Use CMake to install to the distutils build location install_so3g = os.path.dirname( - Path(self.get_ext_fullpath("so3g.libso3g")).resolve().parents[0] + Path(self.get_ext_fullpath("so3g._libso3g")).resolve().parents[0] ) # Fake install directory passed to spt3g cmake. @@ -262,9 +240,6 @@ def build_cmake(self): dlist3g = [ f"-DPython_EXECUTABLE={py_exe}", f"-DPython_INCLUDE_DIRS={py_incl}", - "-DPython_LIBRARIES=''", - "-DPython_RUNTIME_LIBRARY_DIRS=''", - "-DPython_LIBRARY_DIRS=''", f"-DPython_VERSION_MAJOR={py_maj}", f"-DPython_VERSION_MINOR={py_min}", "-DBoost_ARCHITECTURE=-x64", @@ -295,6 +270,13 @@ def build_cmake(self): self.debug, ) + # Move spt3g python directory into place. Remove any stale copy of the + # directory. + install_spt3g_internal = os.path.join(install_so3g, "so3g", "spt3g_internal") + if os.path.isdir(install_spt3g_internal): + shutil.rmtree(install_spt3g_internal) + os.rename(os.path.join(temp_spt3g, "spt3g"), install_spt3g_internal) + build_so3g( topdir, temp_so3g, @@ -306,25 +288,11 @@ def build_cmake(self): self.debug, ) - # Move spt3g python directory into place. Remove any stale copy of the - # directory. - install_spt3g_internal = os.path.join(install_so3g, "so3g", "spt3g_internal") - if os.path.isdir(install_spt3g_internal): - print(f"rm stale: {install_spt3g_internal}") - shutil.rmtree(install_spt3g_internal) - print(f"copy {spt3g_python_dir}, {install_spt3g_internal}") - shutil.copytree(spt3g_python_dir, install_spt3g_internal, symlinks=False) - self.cmake_build_done = True ext_modules = [ - CMakeExtension("so3g.libso3g"), - CMakeExtension("so3g.spt3g_internal.libspt3g-core"), - CMakeExtension("so3g.spt3g_internal.libspt3g-dfmux"), - CMakeExtension("so3g.spt3g_internal.libspt3g-calibration"), - CMakeExtension("so3g.spt3g_internal.libspt3g-gcp"), - CMakeExtension("so3g.spt3g_internal.libspt3g-maps"), + CMakeExtension("so3g._libso3g"), ] # Install the python scripts from spt3g diff --git a/src/G3SuperTimestream.cxx b/src/G3SuperTimestream.cxx index f69228bd..661f77c7 100644 --- a/src/G3SuperTimestream.cxx +++ b/src/G3SuperTimestream.cxx @@ -1176,7 +1176,15 @@ G3SuperTimestreamPtr test_cxx_interface(int nsamps, int first, int second) } -G3_SPLIT_SERIALIZABLE_CODE(G3SuperTimestream); +template void G3SuperTimestream::save( + cereal::PortableBinaryOutputArchive &, unsigned +) const; + +template void G3SuperTimestream::load(cereal::PortableBinaryInputArchive &, unsigned); + +CEREAL_REGISTER_DYNAMIC_INIT(libso3g); + +// G3_SPLIT_SERIALIZABLE_CODE(G3SuperTimestream); static void translate_ValueError(g3supertimestream_exception const& e) { diff --git a/src/main.cxx b/src/main.cxx index 97b761ca..6e299953 100644 --- a/src/main.cxx +++ b/src/main.cxx @@ -39,8 +39,6 @@ bp::object useful_info() { } - - PYBINDINGS("so3g") { bp::def("version", version); bp::def("useful_info", useful_info); @@ -51,7 +49,7 @@ static void* _so3g_import_array() { return NULL; } -BOOST_PYTHON_MODULE(so3g) { +BOOST_PYTHON_MODULE(_libso3g) { _so3g_import_array(); G3ModuleRegistrator::CallRegistrarsFor("so3g"); } diff --git a/test/test_g3super.py b/test/test_g3super.py index e997c378..72a7a2d3 100644 --- a/test/test_g3super.py +++ b/test/test_g3super.py @@ -191,7 +191,7 @@ def test_11_fallback(self): ts = self._get_ts(1, nsamp, sigma=0, dtype='int32') ts.encode() self._readback_compare(ts) - + # Random time vector. n = 200 ts = self._get_ts(1, n, sigma=0, dtype='int32') @@ -241,7 +241,7 @@ def test_30_cpp_interface(self): self.assertTrue(np.all(ts.data[1:] == 0.)) self._readback_compare(ts) del ts - + def test_40_encoding_serialized(self): test_file = 'test_g3super.g3' offsets = { @@ -314,12 +314,12 @@ def test_50_compression(self): del w # Readback - r = so3g.G3IndexedReader(test_file) + r = core.G3Reader(test_file) last = 0 for dtype in ALL_DTYPES: for i in range(2): r.Process(None)[0] - here = r.Tell() + here = r.tell() sizes[dtype].append(here - last) last = here diff --git a/test/test_indexed.py b/test/test_indexed.py index 791653fa..eeb1a82f 100644 --- a/test/test_indexed.py +++ b/test/test_indexed.py @@ -67,7 +67,7 @@ def write_example_file(filename='hk_out.g3'): class TestG3IndexedReader(unittest.TestCase): - """TestCase for testing the so3g.G3IndexedReader, which has seek + """TestCase for testing the spt3g.core.G3Reader, which has seek capabilities to jump to known frames within a .g3 file. """ @@ -80,37 +80,37 @@ def tearDown(self): os.remove(self._file) def test_seek(self): - """Test the Seek/Tell functionality of the G3IndexedReader. We read the + """Test the seek/tell functionality of the G3Reader. We read the first four frames, recording the position of the only Wiring frame in the file - with Tell(). Then we Seek to that location and start reading again, expecting - the first frame after Seek() to be the wiring frame. + with tell(). Then we Seek to that location and start reading again, expecting + the first frame after seek() to be the wiring frame. """ - print("Testing Seek/Tell in G3IndexedReader") - r = so3g.G3IndexedReader(self._file) + print("Testing Seek/Tell in G3Reader") + r = core.G3Reader(self._file) # Limit the number of Process calls, if we hit the end of the file, # then Seek won't work... for i in range(4): - pos = r.Tell() + pos = r.tell() f = r.Process(None)[0] print(" " + str(f.type)) if f.type == core.G3FrameType.Wiring: w_pos = pos print(' Saved wiring frame position: {}'.format(w_pos)) - r.Seek(w_pos) + r.seek(w_pos) # Now that we've seeked, our next frame should be Wiring assert r.Process(None)[0].type == core.G3FrameType.Wiring # Confirm exception is raised if seek at eof. - r = so3g.G3IndexedReader(self._file) + r = core.G3Reader(self._file) while len(r.Process(None)): pass - pos = r.Tell() + pos = r.tell() # Ok to seek to EOF if at EOF. - r.Seek(pos) + r.seek(pos) # No back seeking once there, though. with self.assertRaises(RuntimeError): - r.Seek(0) + r.seek(0) diff --git a/wheels/install_deps_linux.sh b/wheels/install_deps_linux.sh index 7c1d3647..6bf983cd 100755 --- a/wheels/install_deps_linux.sh +++ b/wheels/install_deps_linux.sh @@ -22,7 +22,7 @@ CXX=g++ FC=gfortran CFLAGS="-O3 -fPIC -pthread" -CXXFLAGS="-O3 -fPIC -pthread -std=c++14" +CXXFLAGS="-O3 -fPIC -pthread -std=c++17" FCFLAGS="-O3 -fPIC -pthread" MAKEJ=2 @@ -94,6 +94,7 @@ tar xjf ${boost_pkg} \ ./bootstrap.sh \ --with-python=python3 \ --prefix=${PREFIX} \ + --with-libraries="iostreams,python,regex" \ && ./b2 --layout=tagged --user-config=./tools/build/user-config.jam \ ${pyincl} cxxflags="${CXXFLAGS}" variant=release threading=multi link=shared runtime-link=shared install \ && popd >/dev/null 2>&1 diff --git a/wheels/install_deps_osx.sh b/wheels/install_deps_osx.sh index 4a12fad3..112e2dd6 100755 --- a/wheels/install_deps_osx.sh +++ b/wheels/install_deps_osx.sh @@ -25,7 +25,7 @@ if [ "x${use_gcc}" = "xyes" ]; then CXX=g++-${gcc_version} FC=gfortran-${gcc_version} CFLAGS="-O3 -fPIC" - CXXFLAGS="-O3 -fPIC -std=c++14" + CXXFLAGS="-O3 -fPIC -std=c++17" FCFLAGS="-O3 -fPIC -pthread" OMPFLAGS="-fopenmp" else @@ -35,7 +35,7 @@ else CXX=clang++ #FC="" CFLAGS="-O3 -fPIC" - CXXFLAGS="-O3 -fPIC -std=c++14 -stdlib=libc++" + CXXFLAGS="-O3 -fPIC -std=c++17 -stdlib=libc++" #FCFLAGS="" #OMPFLAGS="" fi @@ -193,6 +193,7 @@ tar xjf ${boost_pkg} \ ./bootstrap.sh \ --with-python=python3 \ --prefix=${PREFIX} \ + --with-libraries="iostreams,python,regex" \ && ./b2 --layout=tagged --user-config=./tools/build/user-config.jam \ ${pyincl} -sNO_LZMA=1 -sNO_ZSTD=1 \ cxxflags="${CXXFLAGS}" ${extra_link} \ diff --git a/wheels/repair_wheel_linux.sh b/wheels/repair_wheel_linux.sh index 0b9d42f7..f9571692 100755 --- a/wheels/repair_wheel_linux.sh +++ b/wheels/repair_wheel_linux.sh @@ -16,7 +16,7 @@ scriptdir=$(pwd) popd >/dev/null 2>&1 # On Linux, we need to add this to LD_LIBRARY_PATH -spt3g_install=$(ls -d ${scriptdir}/../build/lib.*/so3g/spt3g_internal) -export LD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${LD_LIBRARY_PATH} +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install) +export LD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${LD_LIBRARY_PATH} auditwheel repair -w ${dest_dir} ${wheel} diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index a54191b7..c17a2861 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -16,9 +16,12 @@ pushd $(dirname $0) >/dev/null 2>&1 scriptdir=$(pwd) popd >/dev/null 2>&1 -spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g/spt3g) -export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install) +dylib_path="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${DYLD_LIBRARY_PATH} -delocate-listdeps --all ${wheel} \ -&& delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} +# DYLD_LIBRARY_PATH must be set on the command invocation line. See: +# https://github.com/pypa/cibuildwheel/issues/816 +# +DYLD_LIBRARY_PATH=${dylib_path} delocate-listdeps --all ${wheel} && \ +DYLD_LIBRARY_PATH=${dylib_path} delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} diff --git a/wheels/spt3g_disable_tests.patch b/wheels/spt3g_disable_tests.patch deleted file mode 100644 index 18c42aaf..00000000 --- a/wheels/spt3g_disable_tests.patch +++ /dev/null @@ -1,68 +0,0 @@ -diff -urN spt3g_software_orig/cmake/Spt3gBoostPython.cmake spt3g_software_export/cmake/Spt3gBoostPython.cmake ---- spt3g_software_orig/cmake/Spt3gBoostPython.cmake 2024-08-22 10:25:14.077183587 -0700 -+++ spt3g_software_export/cmake/Spt3gBoostPython.cmake 2024-12-11 12:24:37.355444860 -0800 -@@ -1,7 +1,7 @@ - # Locate Python - - if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.12) -- find_package(Python COMPONENTS Interpreter Development) -+ find_package(Python COMPONENTS Interpreter) - else() - find_package(PythonInterp) - find_package(PythonLibs ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) -diff -urN spt3g_software_orig/CMakeLists.txt spt3g_software_export/CMakeLists.txt ---- spt3g_software_orig/CMakeLists.txt 2024-08-22 10:24:59.301256298 -0700 -+++ spt3g_software_export/CMakeLists.txt 2024-12-11 12:24:37.364444816 -0800 -@@ -42,7 +42,7 @@ - - # Raise errors on every warning by default - # (use target-specific options to disable particular warnings) --set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") -+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") - - # Interface library for flags and library dependencies - add_library(spt3g INTERFACE) -diff -urN spt3g_software_orig/core/CMakeLists.txt spt3g_software_export/core/CMakeLists.txt ---- spt3g_software_orig/core/CMakeLists.txt 2024-08-06 11:34:45.598647939 -0700 -+++ spt3g_software_export/core/CMakeLists.txt 2024-12-11 12:24:37.364444816 -0800 -@@ -105,8 +105,8 @@ - add_spt3g_test(quaternions) - add_spt3g_test(timesample) - --add_spt3g_test_program(test -- SOURCE_FILES -- ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamTest.cxx -- ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamMapTest.cxx -- USE_PROJECTS core) -+#add_spt3g_test_program(test -+# SOURCE_FILES -+# ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamTest.cxx -+# ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamMapTest.cxx -+# USE_PROJECTS core) -diff -urN spt3g_software_orig/core/src/dataio.cxx spt3g_software_export/core/src/dataio.cxx ---- spt3g_software_orig/core/src/dataio.cxx 2024-08-06 11:34:45.606647906 -0700 -+++ spt3g_software_export/core/src/dataio.cxx 2024-12-11 12:24:45.732404214 -0800 -@@ -146,8 +146,14 @@ - stream.push(fs); - } else { - // Simple file case -+ const char * bufcheck = getenv("SO3G_FILESYSTEM_BUFFER"); -+ // Use 20MB default -+ size_t so3g_buffer_size = 20971520; -+ if (bufcheck != nullptr) { -+ so3g_buffer_size = (size_t)atol(bufcheck); -+ } - stream.push(boost::iostreams::file_source(path, -- std::ios::binary)); -+ std::ios::binary), so3g_buffer_size); - } - - return fd; -diff -urN spt3g_software_orig/examples/CMakeLists.txt spt3g_software_export/examples/CMakeLists.txt ---- spt3g_software_orig/examples/CMakeLists.txt 2024-08-06 11:34:45.610647890 -0700 -+++ spt3g_software_export/examples/CMakeLists.txt 2024-12-11 12:24:37.365444811 -0800 -@@ -1,2 +1,2 @@ --add_executable(cppexample cppexample.cxx) --target_link_libraries(cppexample core) -+#add_executable(cppexample cppexample.cxx) -+#target_link_libraries(cppexample core) diff --git a/wheels/spt3g_sys_time.patch b/wheels/spt3g_sys_time.patch deleted file mode 100644 index ff4510e0..00000000 --- a/wheels/spt3g_sys_time.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -urN spt3g_software_orig/core/src/G3TimeStamp.cxx spt3g_software/core/src/G3TimeStamp.cxx ---- spt3g_software_orig/core/src/G3TimeStamp.cxx 2025-01-28 20:47:41.412059665 -0800 -+++ spt3g_software/core/src/G3TimeStamp.cxx 2025-01-28 21:09:58.481129822 -0800 -@@ -2,7 +2,7 @@ - #include - - #include --#include -+#include - #include - #include - diff --git a/wheels/test_local_cibuildwheel.sh b/wheels/test_local_cibuildwheel.sh index 1c39fc4c..2b879535 100755 --- a/wheels/test_local_cibuildwheel.sh +++ b/wheels/test_local_cibuildwheel.sh @@ -8,7 +8,7 @@ export CIBW_DEBUG_KEEP_CONTAINER=TRUE export CIBW_BUILD="cp312-manylinux_x86_64" export CIBW_MANYLINUX_X86_64_IMAGE="manylinux2014" export CIBW_BUILD_VERBOSITY=3 -export CIBW_ENVIRONMENT_LINUX="CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14'" +export CIBW_ENVIRONMENT_LINUX="CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++17'" export CIBW_BEFORE_BUILD_LINUX="./wheels/install_deps_linux.sh" export CIBW_REPAIR_WHEEL_COMMAND_LINUX="./wheels/repair_wheel_linux.sh {dest_dir} {wheel}" export CIBW_BEFORE_TEST="export OMP_NUM_THREADS=2" diff --git a/wheels/test_local_macos.sh b/wheels/test_local_macos.sh index 195a7403..90abf51e 100755 --- a/wheels/test_local_macos.sh +++ b/wheels/test_local_macos.sh @@ -50,7 +50,7 @@ export FC=gfortran-14 export CFLAGS="-O3 -fPIC" export FCFLAGS="-O3 -fPIC" # Use the second when building with clang -CXXFLAGS="-O3 -fPIC -std=c++14" +CXXFLAGS="-O3 -fPIC -std=c++17" #CXXFLAGS="-O3 -fPIC -std=c++14 -stdlib=libc++" # Install most dependencies with homebrew, including python-3.9