From: Ole Streicher Date: Thu, 13 Nov 2025 08:15:43 +0000 (+0100) Subject: Import python-fitsio_1.3.0+ds.orig.tar.xz X-Git-Tag: archive/raspbian/1.3.0+ds-2+rpi1~3 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=2dacf1dec3e2293366ef655ecf18a5343e8a5d28;p=python-fitsio.git Import python-fitsio_1.3.0+ds.orig.tar.xz [dgit import orig python-fitsio_1.3.0+ds.orig.tar.xz] --- 2dacf1dec3e2293366ef655ecf18a5343e8a5d28 diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 0000000..a795c89 --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: 8035ed4af26c65303bde38f7d7ba1bf3ff62b9ca +node-date: 2025-11-11T11:34:39-06:00 +describe-name: 1.3.0 +ref-names: HEAD -> master, tag: 1.3.0 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..defee2f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto + +.git_archival.txt export-subst diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5f454fd --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - '*' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..22c4521 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,42 @@ +name: lint + +on: + push: + branches: + - master + pull_request: null + +env: + PY_COLORS: "1" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 + with: + channels: conda-forge + channel-priority: strict + show-channel-urls: true + miniforge-version: latest + + - name: install conda deps + shell: bash -l {0} + run: | + conda list + conda install pre-commit "identify>2.6" + + - name: pre-commit + shell: bash -l {0} + run: | + pre-commit run -a diff --git a/.github/workflows/tests-external-cfitsio.yml b/.github/workflows/tests-external-cfitsio.yml new file mode 100644 index 0000000..45e4f44 --- /dev/null +++ b/.github/workflows/tests-external-cfitsio.yml @@ -0,0 +1,171 @@ +name: tests-external-cfitsio + +on: + push: + branches: + - master + pull_request: null + +env: + PY_COLORS: "1" + # These compiler flags force the tests to fail if arrays are + # accessed at the C level from an unaligned location. + TEST_CFLAGS: "-fsanitize=alignment -fno-sanitize-recover=alignment" + LATEST_CFITSIO_VER: "-4.6.3" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests-external-cfitsio: + name: tests-external-cfitsio + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest] + config: + # 3.44 is the last version that did not support uint64 + - { pyver: "3.8", npver: "1", cftsver: "3440"} + # python 3 string writing fails on all 3.* builds + # so test first 4.* release + - { pyver: "3.8", npver: "1", cftsver: "-4.0.0"} + # 4.1.0 is the first version for which tests pass for uint64 + - { pyver: "3.8", npver: "1", cftsver: "-4.1.0"} + - { pyver: "3.8", npver: "1", cftsver: "latest"} + - { pyver: "3.13", npver: "2.3.0", cftsver: "latest"} + + runs-on: ${{ matrix.os }} + env: + PIP_OPTIONS: "--no-cache-dir --no-deps --no-build-isolation -v" + + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 + with: + python-version: ${{ matrix.config.pyver }} + channels: conda-forge + channel-priority: strict + show-channel-urls: true + miniforge-version: latest + + - name: install conda deps + shell: bash -l {0} + run: | + conda install \ + numpy=${{ matrix.config.npver }} \ + "setuptools-scm>=8" \ + wget \ + make \ + pytest \ + setuptools + conda list + + - name: build external cfitsio + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + mkdir -p cfitsio-external-build + cd cfitsio-external-build + rm -rf * + + if [[ "${{ matrix.config.cftsver }}" == "latest" ]]; then + cftsver=${LATEST_CFITSIO_VER} + else + cftsver="${{ matrix.config.cftsver }}" + fi + + if [[ "${{ matrix.config.cftsver }}" == *3* ]]; then + config_flags="" + else + config_flags="--without-fortran --disable-shared" + fi + + cfitsio_name=cfitsio${cftsver} + wget https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/${cfitsio_name}.tar.gz + cfitsio_dir=`tar -tzf ${cfitsio_name}.tar.gz | sed -n "1,1p" | cut -f1 -d"/"` + tar -xzvf ${cfitsio_name}.tar.gz + cd ${cfitsio_dir} + CFLAGS="-fPIC" ./configure --prefix=$HOME/cfitsio-static-install ${config_flags} + make install -j 4 + cd .. + cd .. + + - name: test non-bundled build + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + pip install ${PIP_OPTIONS} -e . \ + --config-settings="--global-option=--use-system-fitsio" \ + --config-settings="--global-option=--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \ + --config-settings="--global-option=--system-fitsio-libdir=$HOME/cfitsio-static-install/lib" + pytest -vv fitsio + python -c "import fitsio; assert not fitsio.cfitsio_has_bzip2_support()" + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()" + else + python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + fi + + - name: install bzip2 on linux + shell: bash -l {0} + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get install libbz2-dev + + - name: build external cfitsio w/ bzip2 + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + mkdir -p cfitsio-external-build + cd cfitsio-external-build + rm -rf * + + if [[ "${{ matrix.config.cftsver }}" == "latest" ]]; then + cftsver=${LATEST_CFITSIO_VER} + else + cftsver="${{ matrix.config.cftsver }}" + fi + + if [[ "${{ matrix.config.cftsver }}" == *3440* ]]; then + config_flags="--with-bzip2" + else + config_flags="--without-fortran --disable-shared --with-bzip2" + fi + + cfitsio_name=cfitsio${cftsver} + wget https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/${cfitsio_name}.tar.gz + cfitsio_dir=`tar -tzf ${cfitsio_name}.tar.gz | sed -n "1,1p" | cut -f1 -d"/"` + tar -xzvf ${cfitsio_name}.tar.gz + cd ${cfitsio_dir} + CFLAGS="-fPIC" ./configure --prefix=$HOME/cfitsio-static-install ${config_flags} + make install -j 4 + cd .. + cd .. + + - name: test non-bundled build w/ env vars w/ bzip2 + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + pip cache purge + rm -rf build* + find . -name "*.so" -type f -delete + export FITSIO_USE_SYSTEM_FITSIO=1 + export FITSIO_SYSTEM_FITSIO_INCLUDEDIR=$HOME/cfitsio-static-install/include + export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib + pip install ${PIP_OPTIONS} -e . + pytest -vv fitsio + python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()" + else + python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + fi diff --git a/.github/workflows/tests-pypi.yml b/.github/workflows/tests-pypi.yml new file mode 100644 index 0000000..27e56e0 --- /dev/null +++ b/.github/workflows/tests-pypi.yml @@ -0,0 +1,60 @@ +name: tests-pypi + +on: + push: + branches: + - master + pull_request: null + +env: + PY_COLORS: "1" + # These compiler flags force the tests to fail if arrays are + # accessed at the C level from an unaligned location. + TEST_CFLAGS: "-fsanitize=alignment -fno-sanitize-recover=alignment" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests-pypi: + name: tests-pypi + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-22.04] + pyver: ["3.8", "3.13"] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-python@v6 + with: + python-version: '${{ matrix.pyver }}' + + - name: install pip & setuptools + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install pytest + + - name: install bzip2 and other tools on linux + if: contains(matrix.os, 'ubuntu') + run: | + sudo apt-get install libbz2-dev wget make curl libcurl4-openssl-dev + + - name: build fitsio + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + pip install -v -e . + + - name: test fitsio + run: | + pytest -vv fitsio + python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" + python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..122639e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,134 @@ +name: tests + +on: + push: + branches: + - master + pull_request: null + +env: + PY_COLORS: "1" + # These compiler flags force the tests to fail if arrays are + # accessed at the C level from an unaligned location. + TEST_CFLAGS: "-fsanitize=alignment -fno-sanitize-recover=alignment" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + name: tests + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest] + config: + - { pyver: "3.8", npver: "1"} + - { pyver: "3.11", npver: "1.26"} + - { pyver: "3.12", npver: "1.26"} + - { pyver: "3.11", npver: "2.3.0"} + - { pyver: "3.12", npver: "2.3.0"} + - { pyver: "3.13", npver: "2.3.0"} + + runs-on: ${{ matrix.os }} + env: + PIP_OPTIONS: "--no-cache-dir --no-deps --no-build-isolation -v" + + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: conda-incubator/setup-miniconda@835234971496cad1653abb28a638a281cf32541f # v3.2.0 + with: + python-version: ${{ matrix.config.pyver }} + channels: conda-forge + channel-priority: strict + show-channel-urls: true + miniforge-version: latest + + - name: install conda deps + shell: bash -l {0} + run: | + conda install \ + numpy=${{ matrix.config.npver }} \ + "setuptools-scm>=8" \ + wget \ + make \ + pytest \ + setuptools + conda list + + - name: test bundled build + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + pip cache purge + rm -rf build* + rm -rf $HOME/cfitsio-static-install + find . -name "*.so" -type f -delete + pip install ${PIP_OPTIONS} -e . + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + pytest -vv fitsio + python -c "import fitsio; assert not fitsio.cfitsio_has_bzip2_support()" + python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()" + else + pytest -vv fitsio + python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" + python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + fi + + - name: install bzip2 and curl on linux + shell: bash -l {0} + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get install libbz2-dev curl libcurl4-openssl-dev + + - name: test install sdist .gz with no unit tests + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + pip cache purge + rm -rf build* + rm -rf $HOME/cfitsio-static-install + find . -name "*.so" -type f -delete + rm -rf dist + + python setup.py sdist + + pip install ${PIP_OPTIONS} dist/*.tar.gz + cd .. + python -c "import fitsio; assert not fitsio.__version__.startswith('0')" + cd - + pip uninstall fitsio --yes + + - name: test sdist + shell: bash -l {0} + run: | + export CFLAGS="${CFLAGS} ${TEST_CFLAGS}" + + pip cache purge + rm -rf build* + rm -rf $HOME/cfitsio-static-install + find . -name "*.so" -type f -delete + rm -rf dist + + python setup.py sdist + pushd dist/ + + fname=$(ls fitsio*.gz) + tar xvfz "$fname" + dname=$(echo "$fname" | sed 's/\.tar\.gz//') + pushd $dname + + pip install ${PIP_OPTIONS} -e . + pytest -vv fitsio + python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()" + python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()" + + popd + popd diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml new file mode 100644 index 0000000..905db02 --- /dev/null +++ b/.github/workflows/wheel.yml @@ -0,0 +1,237 @@ +# this is based on the wheel workflow in GalSim, but adapted to our needs +name: build wheels and sdist + +on: + workflow_dispatch: + inputs: + ref: + description: 'The git ref to build wheels for. This will trigger a pypi upload.' + default: '' + required: false + type: string + cibw_skip: + description: 'Python versions to skip when building wheels.' + default: 'cp36* cp37* pp* cp38*' + required: false + type: string + pull_request: null + release: + types: + - published + +concurrency: + group: pypi + cancel-in-progress: false + +env: + PYVER: '3.11' + CIBW_SKIP_VAL: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.cibw_skip || 'cp36* cp37* pp* cp38*' }} + +jobs: + linux-manylinux: + name: linux-manylinux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }} + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-python@v6 + with: + python-version: '${{ env.PYVER }}' + + - name: build wheels + uses: pypa/cibuildwheel@v3.2.1 + env: + CIBW_BUILD: "*manylinux*" + CIBW_ARCHS: auto64 + CIBW_SKIP: ${{ env.CIBW_SKIP_VAL }} + # I think yum might always work here. But leave all options available. + CIBW_BEFORE_ALL: yum install -y bzip2-devel || apt-get install libbz2-dev || apk add --upgrade bzip2-dev + + - name: test wheel for python ${{ env.PYVER }} + run: | + pystr='${{ env.PYVER }}' + pystr=${pystr//./} + python -m pip install pip + pip install numpy pytest + pip install ./wheelhouse/*cp${pystr}*.whl + pytest --pyargs fitsio + + - uses: actions/upload-artifact@v5 + with: + name: whl-linux + path: ./wheelhouse/*.whl + + linux-musl: + name: linux-musl + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }} + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-python@v6 + with: + python-version: '${{ env.PYVER }}' + + - name: build wheels + uses: pypa/cibuildwheel@v3.2.1 + env: + CIBW_BUILD: "*musllinux*" + CIBW_ARCHS: auto64 + CIBW_SKIP: ${{ env.CIBW_SKIP_VAL }} + # I think musl always uses apk, but keep all options available. + CIBW_BEFORE_ALL: yum install -y bzip2-devel || apt-get install libbz2-dev || apk add --upgrade bzip2-dev + + - uses: jirutka/setup-alpine@v1 + with: + packages: "bzip2-dev python3 py3-pip py3-numpy" + + - name: test wheel for python + shell: alpine.sh {0} + run: | + python --version + pystr=$(python --version | cut -d' ' -f 2 | cut -d'.' -f 1)$(python --version | cut -d' ' -f 2 | cut -d'.' -f 2) + mkdir test-venv + python3 -m venv test-venv + . test-venv/bin/activate + pip install numpy pytest + pip install ./wheelhouse/*cp${pystr}*musl*.whl + pytest --pyargs fitsio + deactivate + + - uses: actions/upload-artifact@v5 + with: + name: whl-musl + path: ./wheelhouse/*.whl + + osx-intel: + name: osx-intel + runs-on: macos-13 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }} + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-python@v6 + with: + python-version: '${{ env.PYVER }}' + + - name: build wheels + uses: pypa/cibuildwheel@v3.2.1 + env: + CIBW_BUILD: "*macosx*" + CIBW_ARCHS: auto64 + CIBW_SKIP: ${{ env.CIBW_SKIP_VAL }} + # CIBW_BEFORE_ALL: brew install fftw || true + CIBW_ENVIRONMENT: >- + MACOSX_DEPLOYMENT_TARGET=13.0 + + - name: test wheel for python ${{ env.PYVER }} + run: | + pystr='${{ env.PYVER }}' + pystr=${pystr//./} + python -m pip install pip + pip install numpy pytest + pip install ./wheelhouse/*cp${pystr}*.whl + pytest --pyargs fitsio + + - uses: actions/upload-artifact@v5 + with: + name: whl-macos + path: ./wheelhouse/*.whl + + osx-arm: + name: osx-arm + runs-on: macos-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }} + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-python@v6 + with: + python-version: '${{ env.PYVER }}' + + - name: build wheels + uses: pypa/cibuildwheel@v3.2.1 + env: + CIBW_BUILD: "*macosx*" + CIBW_ARCHS: arm64 + CIBW_SKIP: ${{ env.CIBW_SKIP_VAL }} + # CIBW_BEFORE_ALL: brew install llvm libomp fftw eigen + CIBW_ENVIRONMENT: >- + MACOSX_DEPLOYMENT_TARGET=14.7 + + - name: test wheel for python ${{ env.PYVER }} + run: | + pystr='${{ env.PYVER }}' + pystr=${pystr//./} + python -m pip install pip + pip install numpy pytest + pip install ./wheelhouse/*cp${pystr}*.whl + pytest --pyargs fitsio + + - uses: actions/upload-artifact@v5 + with: + name: whl-arm + path: ./wheelhouse/*.whl + + sdist: + name: sdist + needs: [linux-manylinux, linux-musl, osx-intel, osx-arm] + # Just need to build sdist on a single machine + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || '' }} + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-python@v6 + with: + python-version: '${{ env.PYVER }}' + + - name: Install dependencies + run: | + python -m pip install -U pip + pip install -U build + + - name: download wheels + uses: actions/download-artifact@v6 + with: + path: ./wheels + pattern: whl-* + merge-multiple: true + + - name: build sdist + run: | + python -m build --sdist . + ls -l dist + tar tvfz dist/*.tar.gz + + - name: copy wheels to dist + run: | + echo ls -l wheels + ls -l wheels + cp wheels/*.whl dist + echo ls -l dist + ls -l dist + + - name: publish to pypi + uses: pypa/gh-action-pypi-publish@release/v1 + if: github.event_name == 'release' || github.event_name == 'workflow_dispatch' + with: + verbose: true + skip-existing: true + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..04ec0b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,126 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# vscode +.vscode/ + +# stuff for cfitsio +cfitsio-*/cfitsio.pc +cfitsio-*/config.status +cfitsio-*/Makefile + +fitsio/_version.py + +.ruff_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..acb935d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: false + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: monthly + skip: [] + submodules: false + +exclude: '(^cfitsio-.*\/)|(^patches\/.*)|(^fitsio\/test_images\/.*)|(^zlib\/.*)' + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.13.3 + hooks: + - id: ruff-check + args: [ --fix ] + - id: ruff-format + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v21.1.2 + hooks: + - id: clang-format + types_or: [c] + args: [ "-style={BasedOnStyle: llvm, IndentWidth: 4}" ] diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..0b3ef13 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,889 @@ +version 1.3.0 +------------- + +Changes + + - Introduced a `fitsio.NOT_SET` singleton for some + parameters whose defaults are deferred at the Python + level and instead set at the C level by cfitsio. The + image compression parameters now use this singleton. + However, the actual underlying defaults used have not + changed. + - Added a new function `fitsio.cfitsio_is_bundled()` + that detects if the cfitsio library is bundled with + the Python code. + - Enabled the code to track all cfitsio error messages, + including those put onto the internal stack and then + later removed. This feature should help with debugging. + - Added support for uint64 / ULONLONG data for binary tables + and images. You need cfitsio version at least 4.1.0 (or + to use the bundled cfitsio) for this feature to work. + - Installation of fitsio will fail if the `patch` command + line utility is missing. To prevent this, set the environment + variable `FITSIO_FAIL_ON_BAD_PATCHES=false`. + - The C code is now styled uniformly with clang-format. + - Updated bundled cfitsio to 4.6.3. + - Added utilities `cfitsio_has_bzip2_support` and + `cfitsio_has_curl_support` to detect at run-time if cfitsio + was built with these options. + - Added methods `delete_key` and `delete_keys` to HDUs to allow + deleting header keys. + - HDU info loading is now done lazily. + - Changed string handling in python 3 to allow for correct + null-terminated behavior when linking to external builds + of cfitsio at version 4 or larger. + +Bug Fixes + + - Fixed incorrect/unspecified minimum python version, + setting it to `>=3.8`. + - Fixed bug where we attempted to open `mem://` files + as existing files when calling `fitsio.FITS.reopen`. + - Fixed a bug where compression parameters set in filenames + were inconsistently applied, especially when compression + parameters were specified in Python too. Now the code will + raise an exception if a user sets Python keyword compression + parameters while also using filename compression parameters. + However, the `dither_seed` can be set from Python even if other + compression parameters are specified in the filename. + - Fixed bugs in lossless GZIP compression of integer types. See the + new patch `patches/imcompress.c.patch`. See https://github.com/HEASARC/cfitsio/pull/97 + and https://github.com/HEASARC/cfitsio/pull/98 for the upstream PR for the patch. + - Fixed a bug where compression parameters were cached across different HDUs. + - Fixed a bug where writing unsupported image types either did not raise an error + or did not raise the correct error. + - Fixed a bug where rectangular subsets of images were not written properly. + - Fixed automatic detection of bzip2 and curl libraries. + - Fixed handling nan values when reading and writing compressed images. + - Fixed bug in cfitsio where underflows were cast to zero when handling + nan values. See https://github.com/HEASARC/cfitsio/pull/102. + - Fixed bug in cfitsio where overwriting a tile-compressed image fails + when the new values are not lossily compressed but the old values were. + See https://github.com/HEASARC/cfitsio/pull/101. + - Fixed a bug where the FITS HDU properties could go out of sync as header + keys were added, modified, etc. + +version 1.2.8 +------------- + +Changes + + - Arrays passed to cfitsio are now forced to be + aligned via `numpy.require`. This change prevents + failures due to rare instances of unaligned memory + access on certain platforms. No additional copies + of arrays are made unless they are unaligned. As + unaligned arrays are rare, this change should have + minimal performance implications. + +Bug Fixes + + - Fixed error in PyPI uploads. + +version 1.2.7 +------------- + +Changes + + - Replace deprecated `NPY_*` constants (Michał Górny) + +version 1.2.6 +------------- + +Bug Fixes + + - Fix bug parsing header cards with free-form strings + - Fix writing and reading of string columns with length + 1 vectors in numpy 2. + - Fix building against NumPy 2.3.0. + +version 1.2.5 +------------- + +New Features + + - writing images supports the dither_seed keyword, to seed + the random number generator for subtractive dithering + - PyPI now has wheels in addition to sdists + +Bug Fixes + + - Fix bug slicing tables that have TBIT columns + +version 1.2.4 +------------- + +Changes + + - use cfitsio-4.4.1-20240617 which reverts to a free license + +version 1.2.3 +------------- + +Changes + + - bundle cfitsio 4.4.0. + - update manifest to include new cfitsio + +Bug Fixes + + - Reading images with empty slices was returning data + - Using cfitsio 4.4.0 fixes a bug reading HIERARCH+CONTINUE keywords + - zlib subdir not in manifest + +version 1.2.2 +------------- + +Changes + + - Updates for numpy version 2 (Eli Rykoff, ESS) + - setup.py: rename env variable BZIP2 to FITSIO_BZIP2_DIR (Maximillian Bensch) + - Add support for LoongArch (liuxiang) + +version 1.2.1 +------------- + +Changes + + - raise new fitsio.FITSFormatError exception when we detect format + errors in a FITS file. Currently this only raises when extensions + are not properly marked with XTENSION but we can expand this over time + +Bug Fixes + + - Bug not writing compression qlevel when it is set to None/0.0 + This was preventing lossless gzip compression + - work around cfitsio bug when creating HDU for lossless gzip compression, + reopen file instead of just an update hdu list + +version 1.2.0 +-------------- + +Changes + + - move to cfitsio 4 + +Bug Fixes + + - Fixed issue where the patched C fitsio headers were not + first in the include path. + - Added extra header guards and headers to help with compilation issues. + - Fixed builds of the bundled cfitsio code to hide symbols and directly + include the *.o files when linking. + +version 1.1.10 +-------------- + +Changes + +Bug Fixes + + - Fix errors on 32 bit builds where default numpy integer + types were 32 rather than 64 bit assumed by the C code. + - Fix checks for rows being sent to C codes + +version 1.1.9 +------------- + +Changes + + - Row subsets of table data are returned in the order sent by the user + rather than sorted and unique. E.g. rows = [2, 2, 1] will return + data corresponding to that order of rows, including duplicates. + - Removed deprecated `pkg_resources` from tests (M. Becker). + - converted tests to use pytest + - Improved doc strings (N. Tessore) + +Bug Fixes + - Bug updating header string cards was adding new key rather than updating + - Bug gzip compression when not using quantization + +version 1.1.8 +------------- + +Bug Fixes + - Bug in repr of FITS where it only worked the first time it was generated + +version 1.1.7 +------------- + +Bug Fixes + + - Bug converting extension names, should just ensure ascii rather than + subset of ascii + - Bugs in the repr of headers where history keywords had extra spaces. + Also a bug with not upper casing things like 1.0E20 which is the + fits standard + +version 1.1.6 +------------- + +Bug Fixes + + - fixed bug where Updating an existing record in an image header raises an + exception (user ussegliog) + - fix bug append not forwarding arguments to write (Nicolas Tessore) + +version 1.1.5 +--------------------------------- + +Bug Fixes + + - Deal with case that a header keyword has non-ascii + characters and the value is numerical. In this case + we cannot convert the value because fits_read_key + can segfault in some scenarios. We instead return + a string. + - Fixed residual segfaults for header cards with non-ascii + characters. + - HIERARCH keywords are now properly parsed. + - Header keywords with `*`, `#` or `?` in them now properly raise an error + before writing since the generated FITS files cannot be read. + +Changes + + - Non-allowed characters in header keywords are now converted to `_` instead + of `JUNK___...`. + +version 1.1.4 +--------------------------------- + +New Features + + - Moved most testing to GitHub actions (linux, osx). + - Added testing on ppc64le w/ TravisCI (thanks @asellappen) + +Bug Fixes + + - Don't remove BLANK keywords in header clean + - Preserve order of comments in header + +Compatibility changes + + - moved to sing `bool` rather than `np.bool` to be compatible + with numpy 1.2 + +version 1.1.3 +--------------------------------- + +This release moves to cfitsio 3.49, which has bug fixes and now properly +supports reading certain classes of lossless compressed files + +New Features + + - Added keywords to control compression + - qlevel control the quantization level + - qmethod set the quantization method + - hcomp_scale, hcomp_smooth HCOMPRESS specific settings + + A nice result of this is that one can do lossless gzip compression + (setting qlevel=0) and + - Work around some types of garbage characters that might appear + in headers + +BACKWARDS INCOMPATIBLE CHANGES + + - non-ascii junk in headers is replaced by ascii characters to + avoid segmentation faults in the python standard library + when non-unicode characters are detected. This will cause + codes that check for consistency between copied headers + to fail, since the header data is modified. + +Bug Fixes + + - Write integer keywords using the long long support rather than long + - Fix bug where a new file is started and the user can access a + fictional HDU, causing book keeping problems + - Return zero length result when requested rows have + zero length (rainwoodman) + +version 1.1.2 +--------------------------------- + +Bug Fixes + + - Fixed deprecation warnings for extra keyword arguments. + - Fixed SyntaxWarning: "is" with a literal (Michka Popoff) + +version 1.1.1 +--------------------------------- + +Bug Fixes + + - Fix bug in drvrnet.c in printf statement, causing compile + issues on some systems. + +version 1.1.0 +--------------------------------- + +Bumping the minor version due to the update of the cfitsio version + +This reverts to the behavior that compression settings are set as a toggle, +which is the cfitsio convention. The user needs to turn compression on and off +selectively. The alternative behavior, introduced in 1.0.1, broke the mode +where compression is set in the filename, as well as breaking with convention. + +New Features + + - Updated to cfitsio version 3.470 (#261) + - Add ability to stride (step value) when slicing (Dustin Jenkins) + - Add feature to flip along axis when slicing (Dustin Jenkins) + - Feature to ignore image scaling (Dustin Jenkins) + +Bug Fixes + + - Fix error reading with an empty rows argument (rainwoodman) + - Fix bug when reading slice with step, but no start/stop (Mike Jarvis) + - Fix bug with clobber when compression is sent in filename + +Deprecations + + - Removed the use of `**kwargs` in various read/write routines. This + pattern was causing silent bugs. All functions now use explicit + keyword arguments. A warning will be raised in any keyword arguments + are passed. In version `1.2`, this warning will become an error. + +version 1.0.5 +--------------------------------- + +Bug Fixes + + - fixed bug getting `None` keywords + - fixed bug writing 64 bit images (#256, #257) + - fixed HISTORY card value not being read + +version 1.0.4 +--------------------------------- + +New Features + + - support for empty keywords in header, which are supported + by the standard and are used for cosmetic comments + +Bug Fixes + + - Fix for inserting bit columns and appending data with bitcols + - deal with non-standard header values such as NAN + [by returning as strings + - fixed many bugs reading headers; these were a casualty of + the header reading optimizations put in for 1.0.1 + +version 1.0.3 +--------------------------------- + +This is a bug fix release + +Bug Fixes + + - The new header reading code did not deal properly with some + HIERARCH non-standard header key values. + +version 1.0.2 +--------------------------------- + +This is a bug fix release + +Bug Fixes + + - the read_header function was not treating the case_sensitive + keyword properly (Stephen Bailey) + +version 1.0.1 +--------------------------------- + +Backwards Incompatible Changes + + - Support for python 3 strings. + - Support for proper string null termination. This means you can read back exactly + what you wrote. However this departs from previous fitsio which used + the non-standard cfitsio convention of padding strings with spaces. + - Scalar indexing of FITS objects now returns a scalar, consistent + with numpy indexing rules (rainwoodman) + +New Features + + - Installation moved to setuptools from distutils. + - Bundling of cfitsio now done with patches against the upstream + version instead of direct edits to the upstream code. + - Speed improvements for the read_header conveniance function, and + reading of headers in general. + +Bug Fixes + + - CONTINUE in headers are now properly read. Note there is a corner + case that is mis-handled by the underlying cfitsio library. A bug + report has been sent. (thanks for help with Alex Drlica-Wagner + identifying and testing this issue) + - Fixed bug where some long strings were not properly written to headers + - Fixed bug where compression settings for an open FITS object was inherited + from the previous HDU by a new HDU + - Fixed bug where comment strings were lost when setting the value in + a FITSHDR entry + - Fixed bug where get_comment was raising ValueError rather than KeyError + - For py3 need to ensure by hand that strings sizes are greater than 0 + +Deprecations + + - removed `convert` keyword in `FITSRecord` and `FITSHDR` classes. + +version 0.9.12 +--------------------------------- + +New Features + + - Deal properly with undefined value header entries + - can delete rows from a table + - can insert rows with resize() + - can create empty HDU extension for extensions beyond 0 (Felipe Menanteau) + - sanitize string input for py3 + - GZIP_2 compression support (Felipe Menanteau) + - Improvements to python packaging for easier installation. + - Using cfitsio 3.430 now with patches for known bugs + - Now support reading and writing bit columns (Eli Rykoff) + - Can now read CONTINUE keywords in headers. It is currently + treated as a comment; full implementation to come. (Alex Drlica-Wagner) + - Can now use a standard key dict when writing a header key using + the write_key method via `**`, e.g. `write_key(**key_dict)` + (Alex Drlica-Wagner) + - Delete row sets and row ranges using the delete_rows() method + for tables + - Resize tables, adding or removing rows, using the resize() method for + tables + - make write_key usable with standard dictionary using the `**keydict` + style + - allow writing empty HDUs after the first one using + ignore_empty=True to the FITS constructor or + the write convenience function (Felipe Menanteau) + We might make this the default in the future if + it is found to be benign + +Bug Fixes + + - Only raise exception when PLIO u4/u8 is selected now that u1/u2 is supported + in cfitsio (Eli Rykoff) + - link curl library if cfitsio linked to it + - don't require numpy to run setup (Simon Conseil) + - strings with understores in headers, such as `1_000_000` are now not converted to numbers in py3 + - check that the input fields names for tables are unique after converting + to upper case + - link against libm explicitly for compatibility on some systems + +version 0.9.11 +--------------------------------- + +New Features + + - Added trim_strings option to constructor and as keyword for read methods. + If trim_strings=True is set, white space is trimmed from the end + of all string columns upon reading. This was introduced because + cfitsio internally pads strings out with spaces to the full column + width when writing, against the FITS standard. + + - Added read_raw() method to the FITS class, to read the raw underlying data + from the file (Dustin Lang) + +Bug Fixes + + - Fix bug reading hierarch keywords. recent changes to keyword parsing had + broken reading of hierarch keywords + - Fix for strings that look like expressions, e.g. '3-4' which were + being evaluated rather than returned as strings. + - Fix bug for missing key in FITSHDR object using the hdr[key] + notation. Also raise KeyError rather than ValueError + +version 0.9.10 +--------------- + +Bug Fixes + + - Fix variable length string column copying in python 3 + - Fix bug checking for max size in a variable length table column. + - Raise an exception when writing to a table with data + that has shape () + - exit test suite with non-zero exit code if a test fails + +Continuous integration + + - the travis ci now runs unit tests, ignoring those that may fail + when certain libraries/headers are not installed on the users system (for + now this is only bzip2 support) + - only particular pairs of python version/numpy version are tested + +python3 compatibility + + - the compatibility is now built into the code rather than + using 2to3 to modify code at install time. + +Workarounds + + - It turns out that when python, numpy etc. are compiled with gcc 4* + and fitsio is compiled with gcc 5* there is a problem, in some cases, + reading from an array with not aligned memory. This has to do with using + the -O3 optimization flag when compiling cfitsio. For replacing -O3 with + -O2 fixes the issue. This was an issue on linux in both anaconda python2 + and python3. + +version 0.9.9.1 +---------------------------------- + +New tag so that pypi will accept the updated version + +version 0.9.9 +---------------------------------- + +New Features + + - header_start, data_start, data_end now available in the + info dictionary, as well as the new get_offsets() method + to access these in a new dict. + (thanks Dimitri Muna for the initial version of this) + +Bug Fixes + + - Fix bug when writing new COMMENT fields (thanks Alex Drlica-Wagner for + initial fix) + - deal correctly with aligned data in some scenarios + (thanks Ole Streicher) + - use correct data type long for tile_dims_fits in + the set_compression C code. This avoids a crash + on 32 but systems. (thanks Ole Streicher) + - use correct data type npy_int64 for pointer in + get_long_slices (this function is not not correctly + named). Avoids crash on some 32 bit systems. + (thanks Ole Streicher) + - use correct data type npy_int64 for pointer in + PyFITSObject_create_image_hdu, rather than npy_intp. + (thanks Ole Streicher) + +version 0.9.8 +---------------------------------- + +New Features + + - added read_scamp_head function to read the .head files output + by SCAMP and return a FITSHDR object + - reserved header space when creating image and table extensions + and a header is being written. This can improve performance + substantially, especially on distributed file systems. + - When possible write image data at HDU creation. This can + be a big performance improvement, especially on distributed file + systems. + - Support for reading bzipped FITS files. (Dustin Lang) + + - Added option to use the system CFITSIO instead of the bundled one, + by sending --use-system-fitsio. Strongly recommend only use cfitsio + that are as new as the bundled one. Also note the bundled cfitsio + sometimes contains patches that are not yet upstream in an + official cfitsio release + - proper support for reading unsigned images compressed with PLIO. + This is a patch directly on the cfitsio code base. The same + code is in the upstream, but not yet released. + - New method reshape(dims) for images + - When writing into an existing image HDU, and larger dimensions + are required, the image is automatically expanded. + +Bug Fixes + + - Fixed broken boolean fields in new versions of numpy (rainwoodman) Fixed + - bug when image was None (for creating empty first HDU) removed -iarch in + - setup.py for mac OS X. This should + work for versions Mavericks and Snow Leapard (Christopher Bonnett) + - Reading a single string column was failing in some cases, this + has been fixed + - When creating a TableColumnSubset using [cols], the existence + of the columns is checked immediately, rather than waiting for the + check in the read() + - make sure to convert correct endianness when writing during image HDU + creation + - Corrected the repr for single column subsets + - only clean bzero,bscale,bunit from headers for TableHDU + +Dev features + + - added travis ci + +version 0.9.7 +---------------------------------- + +New Features + + - python 3 compatibility + - Adding a new HDU is now near constant time + - Can now create an empty image extension using create_image_hdu + and sending the dims= and dtype= keywords + - Can now write into a sub-section of an existing image using the + start= keyword. + - Can now use a scalar slice for reading images, e.g. + hdu[row1:row2, col] + although this still currently retains the extra dimension + - Use warnings instead of printing to stdout + - IOError is now used to indicate a number of errors that + were previously ValueError + +version 0.9.6 +-------------- + +New Features + + - use cfitsio 3370 to support new tile compression features + - FITSRecord class to encapsulate all the ways one can represent header + records. This is now used internally in the FITSHDR class instead of raw + dicts, but as FITSRecord inherits from dict this should be transparent. + - FITSCard class; inherits from FITSRecord and is a special case for header + card strings + - One can directly add a fits header card string to the FITSHDR object + using add_record + +Bug Fixes + + - use literal_eval instead of eval for evaluating header values (D. Lang) + - If input to write_keys is a FITSHDR, just use it instead of creating a + new FITSHDR object. (D. Lang) + - update existing keys when adding records to FITSHDR, except for + comment and history fields. + - fixed bug with empty string in header card + - deal with cfitsio treating first 4 comments specially + +version 0.9.5 +-------------------------------- + +Note the version 0.9.4 was skipped because some people had been using the +master branch in production, which had version 0.9.4 set. This will allow +automatic version detection to work. In the future master will not have +the next version set until release. + +New Features + + - Re-factored code to use sub-classes for each HDU type. These are called + ImageHDU, TableHDU, and AsciiTableHDU. + - Write and read 32-bit and 64-bit complex table columns + - Write and read boolean table columns (contributed by Dustin Lang) + - Specify tile dimensions for compressed images. + - write_comment and write_history methods added. + - is_compressed() for image HDUs, True if tile compressed. + - added `**keys` to the image hdu reading routines to provide a more uniform + interface for all hdu types + +Bug Fixes + + - Correct appending to COMMENT and HISTORY fields when writing a full + header object. + - Correct conversion of boolean keywords, writing and reading. + - Strip out compression related reserved keywords when writing a + user-provided header. + - Simplified reading string columns in ascii tables so that certain + incorrectly formatted tables from CASUTools are now read accurately. + The change was minimal and did not affect reading well formatted tables, + so seemed worth it. + - Support non-standard TSHORT and TFLOAT columns in ascii tables as + generated by CASUTools. They are non-standard but supporting them + does not seem to break anything (pulled from Simon Walker). + +All changes E. Sheldon except where noted. + +version 0.9.3 +-------------------------- + +New Features + + - Can write lists of arrays and dictionaries of arrays + to fits tables. + - Added iteration over HDUs in FITS class + - Added iteration to the FITSHDU object + - Added iteration to the FITSHDR header object + - added checking that a hdu exists in the file, either + by extension number or name, using the "in" syntax. e.g. + fits=fitsio.FITS(filename) + if 'name' in fits: + data=fits['name'].read() + - added `**keys` to the read_header function + - added get_exttype() to the FITSHDU class + 'BINARY_TBL' 'ASCII_TBL' 'IMAGE_HDU' + - added get_nrows() for binary tables + - added get_colnames() + - added get_filename() + - added get_info() + - added get_nrows() + - added get_vstorage() + - added is_compressed() + - added get_ext() + +minor changes + + - raise error on malformed TDIM + +Backwards incompatible changes + + - renamed some attributes; use the getters instead + - `colnames` -> `_colnames` + - `info` -> `_info` + - `filename` -> `_filename` + - `ext` -> `_ext` + - `vstorage` -> `_vstorage` + - `is_comparessed` -> `_is_compressed` + ( use the getter ) + +Bug Fixes + + - newer numpys (1.6.2) were barfing adding a python float to u4 arrays. + - Give a more clear error message for malformed TDIM header keywords + - fixed bug displaying column info for string array columns in tables + - got cfitsio patch to deal with very large compressed images, which were + not read properly. This is now in the latest cfitsio. + - implemented workaround for bug where numpy declareds 'i8' arrays as type + npy_longlong, which is not correct. + - fixed bug in order of iteration of HDUs + +version 0.9.2 +-------------------------- + +New Features + + - Much faster writing to tables when there are many columns. + - Header object now has a setitem feature + h['item'] = value + - Header stores values now instead of the string rep + - You can force names of fields read from tables to upper + or lower case, either during construction of the FITS object + using or at read time using the lower= and upper= keywords. + +bug fixes + - more sensible data structure for header keywords. Now works in all known + cases when reading and rewriting string fields. + +version 0.9.1 +------------------------- + +New features + + - Added reading of image slices, e.g. `f[ext][2:25, 10:100]` + - Added insert_column(name, data, colnum=) method for HDUs., 2011-11-14 ESS + - Added a verify_checksum() method for HDU objects. 2011-10-24, ESS + - Headers are cleaned of required keyword before writing. E.g. if you have + with fitsio.FITS(file,'rw') as fits: + fits.write(data, header=h) + Keywords like NAXIS, TTYPE* etc are removed. This allows you to read + a header from a fits file and write it to another without clobbering + the required keywords. + + - when accessing a column subset object, more metadata are shown + `f[ext][name]` + - can write None as an image for extension 0, as supported by + the spirit standard. Similarly reading gives None in that case. + - the setup.py is now set up for registering versions to pypi. + +bug fixes + + - Fixed bug that occured sometimes when reading individual columns where a + few bytes were not read. Now using the internal cfitsio buffers more + carefully. + + - Using fits_read_tblbytes when reading full rows fixes a bug that showed + up in a particular file. + + - required header keywords are stripped from input header objects before + writing. + +version 0.9.0 (2011-10-21) +------------------------- + +This is the first "official" release. A patched version of cfitsio 3.28 is now +bundled. This will make it easier for folks to install, and provide a +consistent code base with which to develop. Thanks to Eli Rykoff for +suggesting a bundle. Thanks to Eli and Martin White for helping extensively +with testing. + +On OS X, we now link properly with universal binaries on intel. Thanks to Eli +Rykoff for help with OS X testing and bug fixes. + +New features + + - Write and read variable length columns. When writing a table, any fields + declared "object" ("O" type char) in the input array will be written to a + variable length column. For numbers, this means vectors of varying + length. For strings, it means varying length strings. + + When reading, there are two options. 1) By default the data are read + into fixed length fields with padding to the maximum size in the table + column. This is a "least surprise" approach, since fancy indexing and + other array ops will work as expectd. 2) To save memory, construct the + FITS object with vstorage='object' to store the data as objects. This + storage can also be written back out to a new FITS file with variable + length columns. You can also over-ride the default vstorage when calling + read functions. + + - Write and read ascii tables. cfitsio supports writing scalar 2- and + 4-byte integers, floats and doubles. But for reading only 4-byte integers + and doubles are supported, presumably because of the ambiguity in the + tform fields. Scalar strings are fully supported in both reading and + writing. No array fields are supported for ascii. + + - Append rows to an existing table using the append method. + >>> fits.write_table(data1) + >>> fits[-1].append(data2) + + - Using the new "where" method, you can select rows in a table where an + input expression evaluates to true. The table is scanned row by row + without a large read. This is surprisingly fast, and useful for figuring + out what sections of a large file you want to extract. only requires + enough memory to hold the row indices. + + >>> w=fits[ext].where('x > 3 && y < 25') + >>> data=fits[ext].read(rows=w) + >>> data=fits[ext][w] + + - You can now read rows and columns from a table HDU using slice notation. e.g. + to read row subsets from extension 1 + >>> fits=fitsio.FITS(filename) + >>> data=fits[1][:] + >>> data=fits[1][10:30] + >>> data=fits[1][10:30:2] + + You can also specify a list of rows + >>> rows=[3,8,25] + >>> data=fits[1][rows] + + This is equivalent to + >>> data=fits[1].read(rows=rows) + + To get columns subsets, the notation is similar. The data are read + when the rows are specified. If a sequence of columns is entered, + a recarray is returned, otherwise a simple array. + >>> data=fits[1]['x'][:] + >>> data=fits[1]['x','y'][3:20] + >>> data=fits[1][column_list][row_list] + + + - Added support for EXTVER header keywords. When choosing an HDU by name, + this allows one to select among HDUs that have the same name. Thanks to + Eli Rykoff for suggesting this feature and helping with testing. + + - Name matching for table columns and extension names is not + case-insensitive by default. You can turn on case sensitivity by + constructing the FITS object with case_sensitive=True, or sending + that keyword to the convenience functions read and read_header. + + - Added write_checksum method to the FITSHDU class, which computes the + checksum for the HDU, both the data portion alone (DATASUM keyword) + and the checksum complement for the entire HDU (CHECKSUM). + + - Added an extensive test suite. Use this to run the tests + fitsio.test.test() + + - Added fitsio.cfitsio_version() function, returns the cfitsio + version as a string. + + - added read_slice method, which is used to implement the slice + notation introduced above. + +significant code changes + + - Now using fits_read_tblbytes when reading all rows and columns. This + is just as fast but does not bypass, and thus confuse, the read buffers. + - Removed many direct uses of the internal cfitsio struct objects, + preferring to use provided access functions. This allowed compilation + on older cfitsio that had different struct representations. + +bug fixes + + - too many to list in this early release. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..6d45519 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c50bf66 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include *.txt +include README.md +include cfitsio-4.6.3.tar.gz +include zlib.tar.gz +recursive-include patches * +recursive-include fitsio/test_images * diff --git a/README.md b/README.md new file mode 100644 index 0000000..f3b4d41 --- /dev/null +++ b/README.md @@ -0,0 +1,488 @@ +# fitsio + +[![build wheels/sdist](https://github.com/esheldon/fitsio/actions/workflows/wheel.yml/badge.svg)](https://github.com/esheldon/fitsio/actions/workflows/wheel.yml) [![tests](https://github.com/esheldon/fitsio/workflows/tests/badge.svg)](https://github.com/esheldon/fitsio/actions?query=workflow%3Atests) + +A Python library to read from and write to `FITS` files. + +## Description + +This is a Python extension written in C and Python. Data are read into +numerical Python arrays. + +A version of `cfitsio` is bundled with this package, there is no need to install +your own, nor will this conflict with a version you have installed. + +## Some Features + +- Read from and write to image, binary, and ASCII table extensions. +- Read arbitrary subsets of table columns and rows without loading all the data + to memory. +- Read image subsets without reading the whole image. +- Write subsets to existing images. +- Write and read variable length table columns. +- Read images and tables using slice notation similar to `numpy` arrays. (This is like a more + powerful `memmap`, since it is column-aware for tables.) +- Append rows to an existing table. +- Delete row sets and row ranges, resize tables, or insert rows. +- Query the columns and rows in a table. +- Read and write header keywords. +- Read and write images in tile-compressed format (`RICE`, `GZIP`, `PLIO` ,`HCOMPRESS`). +- Read/write `GZIP` files directly. +- Read unix compress (`.Z`, `.zip`) and bzip2 (`.bz2`) files. +- `TDIM` information is used to return array columns in the correct shape. +- Write and read string table columns, including array columns of arbitrary + shape. +- Read and write complex, bool (logical), unsigned integer, signed bytes types. +- Write checksums into the header and verify them. +- Insert new columns into tables in-place. +- Iterate over rows in a table. Data are buffered for efficiency. +- Python 3 support, including Python 3 strings. + +## Examples + +```python +import fitsio +from fitsio import FITS,FITSHDR + +# Often you just want to quickly read or write data without bothering to +# create a FITS object. In that case, you can use the read and write +# convienience functions. + +# read all data from the first hdu that has data +filename='data.fits' +data = fitsio.read(filename) + +# read a subset of rows and columns from a table +data = fitsio.read(filename, rows=[35,1001], columns=['x','y'], ext=2) + +# read the header +h = fitsio.read_header(filename) +# read both data and header +data,h = fitsio.read(filename, header=True) + +# open the file and write a new binary table extension with the data +# array, which is a numpy array with fields, or "recarray". + +data = np.zeros(10, dtype=[('id','i8'),('ra','f8'),('dec','f8')]) +fitsio.write(filename, data) + +# Write an image to the same file. By default a new extension is +# added to the file. use clobber=True to overwrite an existing file +# instead. To append rows to an existing table, see below. + +fitsio.write(filename, image) + +# +# the FITS class gives the you the ability to explore the data, and gives +# more control +# + +# open a FITS file for reading and explore +fits=fitsio.FITS('data.fits') + +# see what is in here; the FITS object prints itself +print(fits) + +file: data.fits +mode: READONLY +extnum hdutype hduname +0 IMAGE_HDU +1 BINARY_TBL mytable + +# at the python or ipython prompt the fits object will +# print itself +>>> fits +file: data.fits +... etc + +# explore the extensions, either by extension number or +# extension name if available +>>> fits[0] + +file: data.fits +extension: 0 +type: IMAGE_HDU +image info: + data type: f8 + dims: [4096,2048] + +# by name; can also use fits[1] +>>> fits['mytable'] + +file: data.fits +extension: 1 +type: BINARY_TBL +extname: mytable +rows: 4328342 +column info: + i1scalar u1 + f f4 + fvec f4 array[2] + darr f8 array[3,2] + dvarr f8 varray[10] + s S5 + svec S6 array[3] + svar S0 vstring[8] + sarr S2 array[4,3] + +# See bottom for how to get more information for an extension + +# [-1] to refers the last HDU +>>> fits[-1] +... + +# if there are multiple HDUs with the same name, and an EXTVER +# is set, you can use it. Here extver=2 +# fits['mytable',2] + + +# read the image from extension zero +img = fits[0].read() +img = fits[0][:,:] + +# read a subset of the image without reading the whole image +img = fits[0][25:35, 45:55] + + +# read all rows and columns from a binary table extension +data = fits[1].read() +data = fits['mytable'].read() +data = fits[1][:] + +# read a subset of rows and columns. By default uses a case-insensitive +# match. The result retains the names with original case. If columns is a +# sequence, a numpy array with fields, or recarray is returned +data = fits[1].read(rows=[1,5], columns=['index','x','y']) + +# Similar but using slice notation +# row subsets +data = fits[1][10:20] +data = fits[1][10:20:2] +data = fits[1][[1,5,18]] + +# Using EXTNAME and EXTVER values +data = fits['SCI',2][10:20] + +# Slicing with reverse (flipped) striding +data = fits[1][40:25] +data = fits[1][40:25:-5] + +# all rows of column 'x' +data = fits[1]['x'][:] + +# Read a few columns at once. This is more efficient than separate read for +# each column +data = fits[1]['x','y'][:] + +# General column and row subsets. +columns=['index','x','y'] +rows = [1, 5] +data = fits[1][columns][rows] + +# data are returned in the order requested by the user +# and duplicates are preserved +rows = [2, 2, 5] +data = fits[1][columns][rows] + +# iterate over rows in a table hdu +# faster if we buffer some rows, let's buffer 1000 at a time +fits=fitsio.FITS(filename,iter_row_buffer=1000) +for row in fits[1]: + print(row) + +# iterate over HDUs in a FITS object +for hdu in fits: + data=hdu.read() + +# Note dvarr shows type varray[10] and svar shows type vstring[8]. These +# are variable length columns and the number specified is the maximum size. +# By default they are read into fixed-length fields in the output array. +# You can over-ride this by constructing the FITS object with the vstorage +# keyword or specifying vstorage when reading. Sending vstorage='object' +# will store the data in variable size object fields to save memory; the +# default is vstorage='fixed'. Object fields can also be written out to a +# new FITS file as variable length to save disk space. + +fits = fitsio.FITS(filename,vstorage='object') +# OR +data = fits[1].read(vstorage='object') +print(data['dvarr'].dtype) + dtype('object') + + +# you can grab a FITS HDU object to simplify notation +hdu1 = fits[1] +data = hdu1['x','y'][35:50] + +# get rows that satisfy the input expression. See "Row Filtering +# Specification" in the cfitsio manual (note no temporary table is +# created in this case, contrary to the cfitsio docs) +w=fits[1].where("x > 0.25 && y < 35.0") +data = fits[1][w] + +# read the header +h = fits[0].read_header() +print(h['BITPIX']) + -64 + +fits.close() + + +# now write some data +fits = FITS('test.fits','rw') + + +# create a rec array. Note vstr +# is a variable length string +nrows=35 +data = np.zeros(nrows, dtype=[('index','i4'),('vstr','O'),('x','f8'), + ('arr','f4',(3,4))]) +data['index'] = np.arange(nrows,dtype='i4') +data['x'] = np.random.random(nrows) +data['vstr'] = [str(i) for i in xrange(nrows)] +data['arr'] = np.arange(nrows*3*4,dtype='f4').reshape(nrows,3,4) + +# create a new table extension and write the data +fits.write(data) + +# can also be a list of ordinary arrays if you send the names +array_list=[xarray,yarray,namearray] +names=['x','y','name'] +fits.write(array_list, names=names) + +# similarly a dict of arrays +fits.write(dict_of_arrays) +fits.write(dict_of_arrays, names=names) # control name order + +# append more rows to the table. The fields in data2 should match columns +# in the table. missing columns will be filled with zeros +fits[-1].append(data2) + +# insert a new column into a table +fits[-1].insert_column('newcol', data) + +# insert with a specific colnum +fits[-1].insert_column('newcol', data, colnum=2) + +# overwrite rows +fits[-1].write(data) + +# overwrite starting at a particular row. The table will grow if needed +fits[-1].write(data, firstrow=350) + + +# create an image +img=np.arange(2*3,dtype='i4').reshape(2,3) + +# write an image in a new HDU (if this is a new file, the primary HDU) +fits.write(img) + +# write an image with rice compression +fits.write(img, compress='rice') + +# control the compression +fimg=np.random.normal(size=2*3).reshape(2, 3) +fits.write(img, compress='rice', qlevel=16, qmethod='SUBTRACTIVE_DITHER_2') + +# lossless gzip compression for integers or floating point +fits.write(img, compress='gzip', qlevel=None) +fits.write(fimg, compress='gzip', qlevel=None) + +# overwrite the image +fits[ext].write(img2) + +# write into an existing image, starting at the location [300,400] +# the image will be expanded if needed +fits[ext].write(img3, start=[300,400]) + +# change the shape of the image on disk +fits[ext].reshape([250,100]) + +# add checksums for the data +fits[-1].write_checksum() + +# can later verify data integridy +fits[-1].verify_checksum() + +# you can also write a header at the same time. The header can be +# - a simple dict (no comments) +# - a list of dicts with 'name','value','comment' fields +# - a FITSHDR object + +hdict = {'somekey': 35, 'location': 'kitt peak'} +fits.write(data, header=hdict) +hlist = [{'name':'observer', 'value':'ES', 'comment':'who'}, + {'name':'location','value':'CTIO'}, + {'name':'photometric','value':True}] +fits.write(data, header=hlist) +hdr=FITSHDR(hlist) +fits.write(data, header=hdr) + +# you can add individual keys to an existing HDU +fits[1].write_key(name, value, comment="my comment") + +# Write multiple header keys to an existing HDU. Here records +# is the same as sent with header= above +fits[1].write_keys(records) + +# write special COMMENT fields +fits[1].write_comment("observer JS") +fits[1].write_comment("we had good weather") + +# write special history fields +fits[1].write_history("processed with software X") +fits[1].write_history("re-processed with software Y") + +fits.close() + +# using a context, the file is closed automatically after leaving the block +with FITS('path/to/file') as fits: + data = fits[ext].read() + + # you can check if a header exists using "in": + if 'blah' in fits: + data=fits['blah'].read() + if 2 in f: + data=fits[2].read() + +# methods to get more information about extension. For extension 1: +f[1].get_info() # lots of info about the extension +f[1].has_data() # returns True if data is present in extension +f[1].get_extname() +f[1].get_extver() +f[1].get_extnum() # return zero-offset extension number +f[1].get_exttype() # 'BINARY_TBL' or 'ASCII_TBL' or 'IMAGE_HDU' +f[1].get_offsets() # byte offsets (header_start, data_start, data_end) +f[1].is_compressed() # for images. True if tile-compressed +f[1].get_colnames() # for tables +f[1].get_colname(colnum) # for tables find the name from column number +f[1].get_nrows() # for tables +f[1].get_rec_dtype() # for tables +f[1].get_rec_column_descr() # for tables +f[1].get_vstorage() # for tables, storage mechanism for variable + # length columns + +# public attributes you can feel free to change as needed +f[1].lower # If True, lower case colnames on output +f[1].upper # If True, upper case colnames on output +f[1].case_sensitive # if True, names are matched case sensitive +``` + +## Installation + +The easiest way is using `pip` or `conda`. To get the latest release + +```bash +pip install fitsio + +# update fitsio (and everything else) +pip install fitsio --upgrade + +# if pip refuses to update to a newer version +pip install fitsio --upgrade --ignore-installed + +# if you only want to upgrade fitsio +pip install fitsio --no-deps --upgrade --ignore-installed + +# for conda, use conda-forge +conda install -c conda-forge fitsio +``` + +You can also get the latest source tarball release from + +```url +https://pypi.python.org/pypi/fitsio +``` + +or the bleeding edge source from GitHub or use git. To check out +the code for the first time + +```bash +git clone https://github.com/esheldon/fitsio.git +``` + +Or at a later time to update to the latest + +```bash +cd fitsio +git update +``` + +Use `tar xvfz` to unpack the file, enter the `fitsio` directory and type + +```bash +pip install . +``` + +## Requirements + +- python >=3.8 +- a C compiler and build tools like `make`, `patch`, etc. +- numpy (See the note below. Generally, numpy 1.11 or later is better.) + +### Do not use `numpy` 1.10.0 or 1.10.1 + +There is a serious performance regression in `numpy` 1.10 that results +in `fitsio` running tens to hundreds of times slower. A fix may be +forthcoming in a later release. Please comment on GitHub issue +[numpy/issues/6467](https://github.com/numpy/numpy/issues/6467) +here if this has already impacted your work + +## Tests + +The unit tests should all pass for full support. + +```bash +pytest fitsio +``` + +Some tests may fail if certain libraries are not available, such +as bzip2. This failure only implies that bzipped files cannot +be read, without affecting other functionality. + +## Linting and Code Formatting + +We use the `pre-commit` framework for linting and code formatting. To +run the linting and code formatting, use the following command + +```bash +pre-commit run -a +``` + +## Notes on Usage and Features + +### `cfitsio` bundling + +We bundle cfitsio partly because many deployed versions of cfitsio in the +wild do not have support for interesting features like tiled image compression. +Bundling a version that meets our needs is a safe alternative. + +### Array Ordering + +Since numpy uses C order, FITS uses fortran order, we have to write the TDIM +and image dimensions in reverse order, but write the data as is. Then we need +to also reverse the dims as read from the header when creating the numpy dtype, +but read as is. + +### `distutils` vs `setuptools` + +As of version `1.0.0`, `fitsio` has been transitioned to `setuptools` for packaging +and installation. There are many reasons to do this (and to not do this). However, +at a practical level, what this means for you is that you may have trouble uninstalling +older versions with `pip` via `pip uninstall fitsio`. If you do, the best thing to do is +to manually remove the files manually. See this [stackoverflow question](https://stackoverflow.com/questions/402359/how-do-you-uninstall-a-python-package-that-was-installed-using-distutils) +for example. + +### Python 3 Strings + +As of version `1.0.0`, fitsio now supports Python 3 strings natively. This support +means that for Python 3, native strings are read from and written correctly to +FITS files. All byte string columns are treated as ASCII-encoded unicode strings +as well. For FITS files written with a previous version of fitsio, the data +in Python 3 will now come back as a string and not a byte string. Note that this +support is not the same as full unicode support. Internally, fitsio only supports +the ASCII character set. + +## TODO + +- HDU groups: does anyone use these? If so open an issue! diff --git a/fitsio/__init__.py b/fitsio/__init__.py new file mode 100644 index 0000000..2d84ee9 --- /dev/null +++ b/fitsio/__init__.py @@ -0,0 +1,46 @@ +# flake8: noqa +""" +A python library to read and write data to FITS files using cfitsio. +See the docs at https://github.com/esheldon/fitsio for example +usage. +""" + +try: + from ._version import __version__ +except ImportError: + __version__ = None + +from . import fitslib + +from .fitslib import ( + FITS, + read, + read_header, + read_scamp_head, + write, + READONLY, + READWRITE, + NOCOMPRESS, + RICE_1, + GZIP_1, + GZIP_2, + PLIO_1, + HCOMPRESS_1, + NO_DITHER, + SUBTRACTIVE_DITHER_1, + SUBTRACTIVE_DITHER_2, + NOT_SET, +) + +from .header import FITSHDR, FITSRecord, FITSCard +from .hdu import BINARY_TBL, ASCII_TBL, IMAGE_HDU + +from . import util +from .util import ( + cfitsio_version, + FITSRuntimeWarning, + cfitsio_is_bundled, +) +from ._fitsio_wrap import cfitsio_has_bzip2_support, cfitsio_has_curl_support + +from .fits_exceptions import FITSFormatError diff --git a/fitsio/fits_exceptions.py b/fitsio/fits_exceptions.py new file mode 100644 index 0000000..196d241 --- /dev/null +++ b/fitsio/fits_exceptions.py @@ -0,0 +1,11 @@ +class FITSFormatError(Exception): + """ + Format error in FITS file + """ + + def __init__(self, value): + super(FITSFormatError, self).__init__(value) + self.value = value + + def __str__(self): + return str(self.value) diff --git a/fitsio/fitsio_pywrap.c b/fitsio/fitsio_pywrap.c new file mode 100644 index 0000000..5a24dd3 --- /dev/null +++ b/fitsio/fitsio_pywrap.c @@ -0,0 +1,5384 @@ +/* + * fitsio_pywrap.c + * + * This is a CPython wrapper for the cfitsio library. + + Copyright (C) 2011 Erin Sheldon, BNL. erin dot sheldon at gmail dot com + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "fitsio.h" +#include "fitsio2.h" +#include +#include +// #include "fitsio_pywrap_lists.h" +#include + +// this is not defined anywhere in cfitsio except in +// the fits file structure +#define CFITSIO_MAX_ARRAY_DIMS 99 + +// not sure where this is defined in numpy... +#define NUMPY_MAX_DIMS 32 + +// max len of python error message +#define PYFITS_ERRMSG_LEN 1024 + +struct PyFITSObject { + PyObject_HEAD fitsfile *fits; + // we store the python error message here so that we record all error + // messages as they happen. sometimes cfitsio will clear + // the error stack and this removes important debugging info + char pyfits_errmsg[PYFITS_ERRMSG_LEN]; +}; + +// check unicode for python3, string for python2 +static int is_python_string(const PyObject *obj) { +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_Check(obj) || PyBytes_Check(obj); +#else + return PyUnicode_Check(obj) || PyString_Check(obj); +#endif +} + +/* + Ensure all elements of the null terminated string are ascii, replacing + non-ascii characters with a ? +*/ + +static void convert_to_ascii(char *str) { + size_t size = 0, i = 0; + int cval = 0; + + size = strlen(str); + for (i = 0; i < size; i++) { + cval = (int)str[i]; + if (cval < 0 || cval > 127) { + str[i] = '?'; + } + } +} + +/* + Replace non ascii characters with _ +*/ +static int convert_extname_to_ascii(char *str) { + int was_converted = 0; + size_t size = 0, i = 0; + int cval = 0; + + size = strlen(str); + for (i = 0; i < size; i++) { + cval = (int)str[i]; + + if (cval < 0 || cval > 127) { + was_converted = 1; + str[i] = '_'; + } + } + return was_converted; +} + +/* + Replace bad keyword characters (?, *, #) or non-ascii with valid _ + characters. +*/ +static int +convert_keyword_to_allowed_ascii_template_and_nonascii_only(char *str) { + int isbad = 0, was_converted = 0; + size_t size = 0, i = 0; + int cval = 0; + + size = strlen(str); + for (i = 0; i < size; i++) { + cval = (int)str[i]; + + isbad = (cval == '?') || (cval == '*') || (cval == '#'); + + if (isbad || cval < 32 || cval > 126) { + was_converted = 1; + str[i] = '_'; + } + } + return was_converted; +} + +/* +return 1 if a keyword has non-standard FITS keyword characters. +*/ +static int has_invalid_keyword_chars(char *str) { + int isbad = 0; + size_t size = 0, i = 0; + int cval = 0; + + size = strlen(str); + for (i = 0; i < size; i++) { + cval = (int)str[i]; + + isbad = + !((cval >= 'A' && cval <= 'Z') || (cval >= 'a' && cval <= 'z') || + (cval >= '0' && cval <= '9') || (cval == '-') || (cval == '_')); + } + return isbad; +} + +/* + + get a string version of the object. New memory + is allocated and the receiver must clean it up. + +*/ + +// unicode is common to python 2 and 3 +static char *get_unicode_as_string(PyObject *obj) { + PyObject *tmp = NULL; + char *strdata = NULL; + tmp = PyObject_CallMethod(obj, "encode", NULL); + + strdata = strdup(PyBytes_AsString(tmp)); + Py_XDECREF(tmp); + + return strdata; +} + +static char *get_object_as_string(PyObject *obj) { + PyObject *format = NULL; + PyObject *args = NULL; + char *strdata = NULL; + PyObject *tmpobj1 = NULL; + + if (PyUnicode_Check(obj)) { + + strdata = get_unicode_as_string(obj); + + } else { + +#if PY_MAJOR_VERSION >= 3 + + if (PyBytes_Check(obj)) { + strdata = strdup(PyBytes_AsString(obj)); + } else { + PyObject *tmpobj2 = NULL; + format = Py_BuildValue("s", "%s"); + // this is not a string object + args = PyTuple_New(1); + + PyTuple_SetItem(args, 0, obj); + tmpobj2 = PyUnicode_Format(format, args); + tmpobj1 = PyObject_CallMethod(tmpobj2, "encode", NULL); + + Py_XDECREF(args); + Py_XDECREF(tmpobj2); + + strdata = strdup(PyBytes_AsString(tmpobj1)); + Py_XDECREF(tmpobj1); + Py_XDECREF(format); + } + +#else + // convert to a string as needed + if (PyString_Check(obj)) { + strdata = strdup(PyString_AsString(obj)); + } else { + format = Py_BuildValue("s", "%s"); + args = PyTuple_New(1); + + PyTuple_SetItem(args, 0, obj); + tmpobj1 = PyString_Format(format, args); + + strdata = strdup(PyString_AsString(tmpobj1)); + Py_XDECREF(args); + Py_XDECREF(tmpobj1); + Py_XDECREF(format); + } +#endif + } + + return strdata; +} + +static void set_ioerr_string_from_status(int status, + struct PyFITSObject *self) { + char status_str[FLEN_STATUS], errmsg[FLEN_ERRMSG]; + char str_with_newline[1024]; + int nleft; + // sometimes this function is called without an instantiated PyFITSPObject. + // in that case, we use a local buffer and grab all of the information we + // can to set the error string. this edge case means we will sometimes miss + // error messages that cfitsio puts on the stack but then removes later. + char message_if_null[PYFITS_ERRMSG_LEN]; + + // pointer to the error message buffer we are using + char *message; + + // always init error message to zero length string + // init of self->pyfits_errmsg is done when object is created + message_if_null[0] = '\0'; + + if (self != NULL) { + message = &(self->pyfits_errmsg[0]); + } else { + message = &(message_if_null[0]); + } + + // the number of characters left is the number of characters not + // used in the message variable minus one. We remove one for the + // C null termination character. So if the variable is PYFITS_ERRMSG_LEN + // characters long, then nleft is PYFITS_ERRMSG_LEN - 1. + // This works because: + // - strncat always adds the null termination character + // - strlen returns the number of characters up to the first null (and + // does not include the null itself in the total count). + // Thus we never have to account for the null ourselves except in the + // initial number of characters we have to use for holding error messages + // (which we set to one less than the total storage we have so strncat + // always has room for the null.) + nleft = PYFITS_ERRMSG_LEN - strlen(message) - 1; + + if (status) { + fits_get_errstatus(status, status_str); /* get the error description */ + + sprintf(str_with_newline, "FITSIO status = %d: %s\n", status, + status_str); + if (nleft >= strlen(str_with_newline)) { + strncat(message, str_with_newline, nleft); + nleft -= strlen(str_with_newline); + } + + while (nleft > 0 && + fits_read_errmsg(errmsg)) { /* get error stack messages */ + sprintf(str_with_newline, "%s\n", errmsg); + if (nleft >= strlen(str_with_newline)) { + strncat(message, str_with_newline, nleft); + nleft -= strlen(str_with_newline); + } else { + break; + } + } + PyErr_SetString(PyExc_IOError, message); + } + return; +} + +/* + string list helper functions +*/ + +struct stringlist { + size_t size; + char **data; +}; + +static struct stringlist *stringlist_new(void) { + struct stringlist *slist = NULL; + + slist = malloc(sizeof(struct stringlist)); + slist->size = 0; + slist->data = NULL; + return slist; +} +// push a copy of the string onto the string list +static void stringlist_push(struct stringlist *slist, const char *str) { + size_t newsize = 0; + size_t i = 0; + + newsize = slist->size + 1; + slist->data = realloc(slist->data, sizeof(char *) * newsize); + slist->size += 1; + + i = slist->size - 1; + + slist->data[i] = strdup(str); +} + +static void stringlist_push_size(struct stringlist *slist, size_t slen) { + size_t newsize = 0; + size_t i = 0; + + newsize = slist->size + 1; + slist->data = realloc(slist->data, sizeof(char *) * newsize); + slist->size += 1; + + i = slist->size - 1; + + slist->data[i] = calloc(slen + 1, sizeof(char)); + // slist->data[i] = malloc(sizeof(char)*(slen+1)); + // memset(slist->data[i], 0, slen+1); +} +static struct stringlist *stringlist_delete(struct stringlist *slist) { + if (slist != NULL) { + size_t i = 0; + if (slist->data != NULL) { + for (i = 0; i < slist->size; i++) { + free(slist->data[i]); + } + } + free(slist->data); + free(slist); + } + return NULL; +} + +/* +static void stringlist_print(struct stringlist* slist) { + size_t i=0; + if (slist == NULL) { + return; + } + for (i=0; isize; i++) { + printf(" slist[%ld]: %s\n", i, slist->data[i]); + } +} +*/ + +static int stringlist_addfrom_listobj(struct stringlist *slist, + PyObject *listObj, const char *listname) { + size_t size = 0, i = 0; + char *tmpstr = NULL; + + if (!PyList_Check(listObj)) { + PyErr_Format(PyExc_ValueError, "Expected a list for %s.", listname); + return 1; + } + size = PyList_Size(listObj); + + for (i = 0; i < size; i++) { + PyObject *tmp = PyList_GetItem(listObj, i); + if (!is_python_string(tmp)) { + PyErr_Format(PyExc_ValueError, "Expected only strings in %s list.", + listname); + return 1; + } + tmpstr = get_object_as_string(tmp); + stringlist_push(slist, tmpstr); + free(tmpstr); + } + return 0; +} + +static void add_double_to_dict(PyObject *dict, const char *key, double value) { + PyObject *tobj = NULL; + tobj = PyFloat_FromDouble(value); + PyDict_SetItemString(dict, key, tobj); + Py_XDECREF(tobj); +} + +static void add_long_to_dict(PyObject *dict, const char *key, long value) { + PyObject *tobj = NULL; + tobj = PyLong_FromLong(value); + PyDict_SetItemString(dict, key, tobj); + Py_XDECREF(tobj); +} + +static void add_long_long_to_dict(PyObject *dict, const char *key, + long long value) { + PyObject *tobj = NULL; + tobj = PyLong_FromLongLong(value); + PyDict_SetItemString(dict, key, tobj); + Py_XDECREF(tobj); +} + +static void add_string_to_dict(PyObject *dict, const char *key, + const char *str) { + PyObject *tobj = NULL; + tobj = Py_BuildValue("s", str); + PyDict_SetItemString(dict, key, tobj); + Py_XDECREF(tobj); +} + +static void add_none_to_dict(PyObject *dict, const char *key) { + PyDict_SetItemString(dict, key, Py_None); + Py_XINCREF(Py_None); +} +static void add_true_to_dict(PyObject *dict, const char *key) { + PyDict_SetItemString(dict, key, Py_True); + Py_XINCREF(Py_True); +} +static void add_false_to_dict(PyObject *dict, const char *key) { + PyDict_SetItemString(dict, key, Py_False); + Py_XINCREF(Py_False); +} + +/* +static +void append_long_to_list(PyObject* list, long value) { + PyObject* tobj=NULL; + tobj=PyLong_FromLong(value); + PyList_Append(list, tobj); + Py_XDECREF(tobj); +} +*/ + +static void append_long_long_to_list(PyObject *list, long long value) { + PyObject *tobj = NULL; + tobj = PyLong_FromLongLong(value); + PyList_Append(list, tobj); + Py_XDECREF(tobj); +} + +/* +static +void append_string_to_list(PyObject* list, const char* str) { + PyObject* tobj=NULL; + tobj=Py_BuildValue("s",str); + PyList_Append(list, tobj); + Py_XDECREF(tobj); +} +*/ + +static int PyFITSObject_init(struct PyFITSObject *self, PyObject *args, + PyObject *kwds) { + char *filename; + int mode; + int status = 0; + int create = 0; + + // init the error message to an empty string + self->pyfits_errmsg[0] = '\0'; + + if (!PyArg_ParseTuple(args, (char *)"sii", &filename, &mode, &create)) { + return -1; + } + + if (create) { + // create and open + if (fits_create_file(&self->fits, filename, &status)) { + set_ioerr_string_from_status(status, self); + return -1; + } + } else { + if (fits_open_file(&self->fits, filename, mode, &status)) { + set_ioerr_string_from_status(status, self); + return -1; + } + } + + return 0; +} + +static PyObject *PyFITSObject_repr(struct PyFITSObject *self) { + + if (self->fits != NULL) { + int status = 0; + char filename[FLEN_FILENAME]; + char repr[2056]; + + if (fits_file_name(self->fits, filename, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + sprintf(repr, "fits file: %s", filename); + return Py_BuildValue("s", repr); + } else { + return Py_BuildValue("s", "none"); + } +} + +static PyObject *PyFITSObject_filename(struct PyFITSObject *self) { + + if (self->fits != NULL) { + int status = 0; + char filename[FLEN_FILENAME]; + PyObject *fnameObj = NULL; + if (fits_file_name(self->fits, filename, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + fnameObj = Py_BuildValue("s", filename); + return fnameObj; + } else { + PyErr_SetString(PyExc_ValueError, + "file is not open, cannot determine name"); + return NULL; + } +} + +static PyObject *PyFITSObject_close(struct PyFITSObject *self) { + int status = 0; + if (fits_close_file(self->fits, &status)) { + self->fits = NULL; + /* + set_ioerr_string_from_status(status, self); + return NULL; + */ + } + self->fits = NULL; + Py_RETURN_NONE; +} + +static void PyFITSObject_dealloc(struct PyFITSObject *self) { + int status = 0; + fits_close_file(self->fits, &status); +#if PY_MAJOR_VERSION >= 3 + // introduced in python 2.6 + Py_TYPE(self)->tp_free((PyObject *)self); +#else + // old way, removed in python 3 + self->ob_type->tp_free((PyObject *)self); +#endif +} + +// this will need to be updated for array string columns. +// I'm using a tcolumn* here, could cause problems +static long get_groupsize(tcolumn *colptr) { + long gsize = 0; + if (colptr->tdatatype == TSTRING) { + // gsize = colptr->twidth; + gsize = colptr->trepeat; + } else { + gsize = colptr->twidth * colptr->trepeat; + } + return gsize; +} +static npy_int64 *get_int64_from_array(PyArrayObject *arr, npy_intp *ncols) { + + npy_int64 *colnums; + int npy_type = 0, check = 0; + + if (!PyArray_Check(arr)) { + PyErr_SetString(PyExc_TypeError, "int64 array must be an array."); + return NULL; + } + + npy_type = PyArray_TYPE(arr); + + // on some platforms, creating an 'i8' array gives it a longlong + // dtype. Just make sure it is 8 bytes + check = + (npy_type == NPY_INT64) | + (npy_type == NPY_LONGLONG && sizeof(npy_longlong) == sizeof(npy_int64)); + if (!check) { + PyErr_Format(PyExc_TypeError, + "array must be an int64 array (%d), got %d.", NPY_INT64, + npy_type); + return NULL; + } + if (!PyArray_ISCONTIGUOUS(arr)) { + PyErr_SetString(PyExc_TypeError, "int64 array must be a contiguous."); + return NULL; + } + + colnums = PyArray_DATA(arr); + *ncols = PyArray_SIZE(arr); + + return colnums; +} + +// move hdu by name and possibly version, return the hdu number +static PyObject *PyFITSObject_movnam_hdu(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdutype = ANY_HDU; // means we don't care if its image or table + char *extname = NULL; + int extver = 0; // zero means it is ignored + int hdunum = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"isi", &hdutype, &extname, &extver)) { + return NULL; + } + + if (fits_movnam_hdu(self->fits, hdutype, extname, extver, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + fits_get_hdu_num(self->fits, &hdunum); + return PyLong_FromLong((long)hdunum); +} + +static PyObject *PyFITSObject_movabs_hdu(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0, hdutype = 0; + int status = 0; + PyObject *hdutypeObj = NULL; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"i", &hdunum)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + hdutypeObj = PyLong_FromLong((long)hdutype); + return hdutypeObj; +} + +// get info for the specified HDU +static PyObject *PyFITSObject_get_hdu_info(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0, hdutype = 0, ext = 0, ignore_scaling = FALSE; + int status = 0, tstatus = 0, is_compressed = 0; + PyObject *dict = NULL; + + char extname[FLEN_VALUE]; + char hduname[FLEN_VALUE]; + int extver = 0, hduver = 0; + + long long header_start; + long long data_start; + long long data_end; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"ii", &hdunum, &ignore_scaling)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (ignore_scaling == TRUE && + fits_set_bscale(self->fits, 1.0, 0.0, &status)) { + return NULL; + } + + dict = PyDict_New(); + ext = hdunum - 1; + + add_long_to_dict(dict, "hdunum", (long)hdunum); + add_long_to_dict(dict, "extnum", (long)ext); + add_long_to_dict(dict, "hdutype", (long)hdutype); + + tstatus = 0; + if (fits_read_key(self->fits, TSTRING, "EXTNAME", extname, NULL, + &tstatus) == 0) { + convert_extname_to_ascii(extname); + add_string_to_dict(dict, "extname", extname); + } else { + add_string_to_dict(dict, "extname", ""); + } + + tstatus = 0; + if (fits_read_key(self->fits, TSTRING, "HDUNAME", hduname, NULL, + &tstatus) == 0) { + convert_extname_to_ascii(hduname); + add_string_to_dict(dict, "hduname", hduname); + } else { + add_string_to_dict(dict, "hduname", ""); + } + + tstatus = 0; + if (fits_read_key(self->fits, TINT, "EXTVER", &extver, NULL, &tstatus) == + 0) { + add_long_to_dict(dict, "extver", (long)extver); + } else { + add_long_to_dict(dict, "extver", (long)0); + } + + tstatus = 0; + if (fits_read_key(self->fits, TINT, "HDUVER", &hduver, NULL, &tstatus) == + 0) { + add_long_to_dict(dict, "hduver", (long)hduver); + } else { + add_long_to_dict(dict, "hduver", (long)0); + } + + tstatus = 0; + is_compressed = fits_is_compressed_image(self->fits, &tstatus); + add_long_to_dict(dict, "is_compressed_image", (long)is_compressed); + + // get byte offsets + if (0 == fits_get_hduaddrll(self->fits, &header_start, &data_start, + &data_end, &tstatus)) { + add_long_long_to_dict(dict, "header_start", (long)header_start); + add_long_long_to_dict(dict, "data_start", (long)data_start); + add_long_long_to_dict(dict, "data_end", (long)data_end); + } else { + add_long_long_to_dict(dict, "header_start", -1); + add_long_long_to_dict(dict, "data_start", -1); + add_long_long_to_dict(dict, "data_end", -1); + } + + int ndims = 0; + int maxdim = CFITSIO_MAX_ARRAY_DIMS; + LONGLONG dims[CFITSIO_MAX_ARRAY_DIMS]; + if (hdutype == IMAGE_HDU) { + // move this into it's own func + int tstatus = 0; + int bitpix = 0; + int bitpix_equiv = 0; + char comptype[20]; + PyObject *dimsObj = PyList_New(0); + int i = 0; + + // if (fits_read_imghdrll(self->fits, maxdim, simple_p, &bitpix, &ndims, + // dims, pcount_p, gcount_p, extend_p, &status)) + // { + if (fits_get_img_paramll(self->fits, maxdim, &bitpix, &ndims, dims, + &tstatus)) { + add_string_to_dict(dict, "error", + "could not determine image parameters"); + } else { + add_long_to_dict(dict, "ndims", (long)ndims); + add_long_to_dict(dict, "img_type", (long)bitpix); + + if (ignore_scaling == TRUE) { + // Get the raw type if scaling is being ignored. + fits_get_img_type(self->fits, &bitpix_equiv, &status); + } else { + fits_get_img_equivtype(self->fits, &bitpix_equiv, &status); + } + + add_long_to_dict(dict, "img_equiv_type", (long)bitpix_equiv); + + tstatus = 0; + if (fits_read_key(self->fits, TSTRING, "ZCMPTYPE", comptype, NULL, + &tstatus) == 0) { + convert_to_ascii(comptype); + add_string_to_dict(dict, "comptype", comptype); + } else { + add_none_to_dict(dict, "comptype"); + } + + for (i = 0; i < ndims; i++) { + append_long_long_to_list(dimsObj, (long long)dims[i]); + } + PyDict_SetItemString(dict, "dims", dimsObj); + Py_XDECREF(dimsObj); + } + + } else if (hdutype == BINARY_TBL) { + int tstatus = 0; + LONGLONG nrows = 0; + int ncols = 0; + PyObject *colinfo = PyList_New(0); + int i = 0, j = 0; + + fits_get_num_rowsll(self->fits, &nrows, &tstatus); + fits_get_num_cols(self->fits, &ncols, &tstatus); + add_long_long_to_dict(dict, "nrows", (long long)nrows); + add_long_to_dict(dict, "ncols", (long)ncols); + + { + PyObject *d = NULL; + tcolumn *col = NULL; + struct stringlist *names = NULL; + struct stringlist *tforms = NULL; + names = stringlist_new(); + tforms = stringlist_new(); + + for (i = 0; i < ncols; i++) { + stringlist_push_size(names, 70); + stringlist_push_size(tforms, 70); + } + // just get the names: no other way to do it! + fits_read_btblhdrll(self->fits, ncols, NULL, NULL, names->data, + tforms->data, NULL, NULL, NULL, &tstatus); + + for (i = 0; i < ncols; i++) { + d = PyDict_New(); + int type = 0; + LONGLONG repeat = 0; + LONGLONG width = 0; + + convert_to_ascii(names->data[i]); + add_string_to_dict(d, "name", names->data[i]); + convert_to_ascii(tforms->data[i]); + add_string_to_dict(d, "tform", tforms->data[i]); + + fits_get_coltypell(self->fits, i + 1, &type, &repeat, &width, + &tstatus); + add_long_to_dict(d, "type", (long)type); + add_long_long_to_dict(d, "repeat", (long long)repeat); + add_long_long_to_dict(d, "width", (long long)width); + + fits_get_eqcoltypell(self->fits, i + 1, &type, &repeat, &width, + &tstatus); + add_long_to_dict(d, "eqtype", (long)type); + + tstatus = 0; + if (fits_read_tdimll(self->fits, i + 1, maxdim, &ndims, dims, + &tstatus)) { + add_none_to_dict(d, "tdim"); + } else { + PyObject *dimsObj = PyList_New(0); + for (j = 0; j < ndims; j++) { + append_long_long_to_list(dimsObj, (long long)dims[j]); + } + + PyDict_SetItemString(d, "tdim", dimsObj); + Py_XDECREF(dimsObj); + } + + // using the struct, could cause problems + // actually, we can use ffgcprll to get this info, but will + // be redundant with some others above + col = &self->fits->Fptr->tableptr[i]; + add_double_to_dict(d, "tscale", col->tscale); + add_double_to_dict(d, "tzero", col->tzero); + + PyList_Append(colinfo, d); + Py_XDECREF(d); + } + names = stringlist_delete(names); + tforms = stringlist_delete(tforms); + + PyDict_SetItemString(dict, "colinfo", colinfo); + Py_XDECREF(colinfo); + } + } else { + int tstatus = 0; + LONGLONG nrows = 0; + int ncols = 0; + PyObject *colinfo = PyList_New(0); + int i = 0, j = 0; + + fits_get_num_rowsll(self->fits, &nrows, &tstatus); + fits_get_num_cols(self->fits, &ncols, &tstatus); + add_long_long_to_dict(dict, "nrows", (long long)nrows); + add_long_to_dict(dict, "ncols", (long)ncols); + + { + tcolumn *col = NULL; + struct stringlist *names = NULL; + struct stringlist *tforms = NULL; + names = stringlist_new(); + tforms = stringlist_new(); + + for (i = 0; i < ncols; i++) { + stringlist_push_size(names, 70); + stringlist_push_size(tforms, 70); + } + // just get the names: no other way to do it! + + // rowlen nrows + fits_read_atblhdrll(self->fits, ncols, NULL, NULL, + // tfields tbcol units + NULL, names->data, NULL, tforms->data, NULL, + // extname + NULL, &tstatus); + + for (i = 0; i < ncols; i++) { + PyObject *d = PyDict_New(); + int type = 0; + LONGLONG repeat = 0; + LONGLONG width = 0; + + convert_to_ascii(names->data[i]); + add_string_to_dict(d, "name", names->data[i]); + convert_to_ascii(tforms->data[i]); + add_string_to_dict(d, "tform", tforms->data[i]); + + fits_get_coltypell(self->fits, i + 1, &type, &repeat, &width, + &tstatus); + add_long_to_dict(d, "type", (long)type); + add_long_long_to_dict(d, "repeat", (long long)repeat); + add_long_long_to_dict(d, "width", (long long)width); + + fits_get_eqcoltypell(self->fits, i + 1, &type, &repeat, &width, + &tstatus); + add_long_to_dict(d, "eqtype", (long)type); + + tstatus = 0; + if (fits_read_tdimll(self->fits, i + 1, maxdim, &ndims, dims, + &tstatus)) { + add_none_to_dict(dict, "tdim"); + } else { + PyObject *dimsObj = PyList_New(0); + for (j = 0; j < ndims; j++) { + append_long_long_to_list(dimsObj, (long long)dims[j]); + } + + PyDict_SetItemString(d, "tdim", dimsObj); + Py_XDECREF(dimsObj); + } + + // using the struct, could cause problems + // actually, we can use ffgcprll to get this info, but will + // be redundant with some others above + col = &self->fits->Fptr->tableptr[i]; + add_double_to_dict(d, "tscale", col->tscale); + add_double_to_dict(d, "tzero", col->tzero); + + PyList_Append(colinfo, d); + Py_XDECREF(d); + } + names = stringlist_delete(names); + tforms = stringlist_delete(tforms); + + PyDict_SetItemString(dict, "colinfo", colinfo); + Py_XDECREF(colinfo); + } + } + return dict; +} + +// get info for the specified HDU +static PyObject *PyFITSObject_get_hdu_name_version(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0, hdutype = 0; + int status = 0; + + char extname[FLEN_VALUE]; + int extver = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"i", &hdunum)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + status = 0; + if (fits_read_key(self->fits, TINT, "EXTVER", &extver, NULL, &status) != + 0) { + extver = 0; + } + + status = 0; + if (fits_read_key(self->fits, TSTRING, "EXTNAME", extname, NULL, &status) == + 0) { + return Py_BuildValue("si", extname, extver); + } else { + return Py_BuildValue("si", "", extver); + } +} + +// this is the parameter that goes in the type for fits_write_col +static int npy_to_fits_table_type(int npy_dtype, int write_bitcols) { + + char mess[255]; + switch (npy_dtype) { + case NPY_BOOL: + if (write_bitcols) { + return TBIT; + } else { + return TLOGICAL; + } + case NPY_UINT8: + return TBYTE; + case NPY_INT8: + return TSBYTE; + case NPY_UINT16: + return TUSHORT; + case NPY_INT16: + return TSHORT; + case NPY_UINT32: + if (sizeof(unsigned int) == sizeof(npy_uint32)) { + return TUINT; + } else if (sizeof(unsigned long) == sizeof(npy_uint32)) { + return TULONG; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 4 byte unsigned integer type"); + return -9999; + } + case NPY_INT32: + if (sizeof(int) == sizeof(npy_int32)) { + return TINT; + } else if (sizeof(long) == sizeof(npy_int32)) { + return TLONG; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 4 byte integer type"); + return -9999; + } + + case NPY_INT64: + if (sizeof(long long) == sizeof(npy_int64)) { + return TLONGLONG; + } else if (sizeof(long) == sizeof(npy_int64)) { + return TLONG; + } else if (sizeof(int) == sizeof(npy_int64)) { + return TINT; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 8 byte integer type"); + return -9999; + } +#ifdef TULONGLONG + case NPY_UINT64: + if (sizeof(unsigned long long) == sizeof(npy_uint64)) { + return TULONGLONG; + } else if (sizeof(unsigned long) == sizeof(npy_uint64)) { + return TULONG; + } else if (sizeof(unsigned int) == sizeof(npy_uint64)) { + return TUINT; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 8 byte unsigned integer type"); + return -9999; + } +#else + case NPY_UINT64: + PyErr_SetString(PyExc_TypeError, + "Unsigned 8 byte integer images are not " + "supported by the FITS standard"); + return -9999; +#endif + + case NPY_FLOAT32: + return TFLOAT; + case NPY_FLOAT64: + return TDOUBLE; + + case NPY_COMPLEX64: + return TCOMPLEX; + case NPY_COMPLEX128: + return TDBLCOMPLEX; + + case NPY_STRING: + return TSTRING; + + default: + sprintf(mess, "Unsupported numpy table datatype %d", npy_dtype); + PyErr_SetString(PyExc_TypeError, mess); + return -9999; + } + + return 0; +} + +static int npy_to_fits_image_types(int npy_dtype, int *fits_img_type, + int *fits_datatype) { + + char mess[255]; + switch (npy_dtype) { + case NPY_UINT8: + *fits_img_type = BYTE_IMG; + *fits_datatype = TBYTE; + break; + case NPY_INT8: + *fits_img_type = SBYTE_IMG; + *fits_datatype = TSBYTE; + break; + case NPY_UINT16: + *fits_img_type = USHORT_IMG; + *fits_datatype = TUSHORT; + break; + case NPY_INT16: + *fits_img_type = SHORT_IMG; + *fits_datatype = TSHORT; + break; + + case NPY_UINT32: + //*fits_img_type = ULONG_IMG; + if (sizeof(unsigned short) == sizeof(npy_uint32)) { + *fits_img_type = USHORT_IMG; + *fits_datatype = TUSHORT; + } else if (sizeof(unsigned int) == sizeof(npy_uint32)) { + // there is no UINT_IMG, so use ULONG_IMG + *fits_img_type = ULONG_IMG; + *fits_datatype = TUINT; + } else if (sizeof(unsigned long) == sizeof(npy_uint32)) { + *fits_img_type = ULONG_IMG; + *fits_datatype = TULONG; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 4 byte unsigned integer type"); + *fits_datatype = -9999; + return 1; + } + break; + + case NPY_INT32: + //*fits_img_type = LONG_IMG; + if (sizeof(short) == sizeof(npy_int32)) { + *fits_img_type = SHORT_IMG; + *fits_datatype = TINT; + } else if (sizeof(int) == sizeof(npy_int32)) { + // there is no UINT_IMG, so use ULONG_IMG + *fits_img_type = LONG_IMG; + *fits_datatype = TINT; + } else if (sizeof(long) == sizeof(npy_int32)) { + *fits_img_type = LONG_IMG; + *fits_datatype = TLONG; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 4 byte integer type"); + *fits_datatype = -9999; + return 1; + } + break; + + case NPY_INT64: + if (sizeof(LONGLONG) == sizeof(npy_int64)) { + *fits_img_type = LONGLONG_IMG; + *fits_datatype = TLONGLONG; + } else if (sizeof(long) == sizeof(npy_int64)) { + *fits_img_type = LONG_IMG; + *fits_datatype = TLONG; + } else if (sizeof(int) == sizeof(npy_int64)) { + // there is no UINT_IMG, so use ULONG_IMG + *fits_img_type = LONG_IMG; + *fits_datatype = TINT; + } else if (sizeof(long long) == sizeof(npy_int64)) { + // we don't expect to get here + *fits_img_type = LONGLONG_IMG; + *fits_datatype = TLONGLONG; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 8 byte integer type"); + *fits_datatype = -9999; + return 1; + } + break; + +#ifdef TULONGLONG + case NPY_UINT64: + if (sizeof(ULONGLONG) == sizeof(npy_uint64)) { + *fits_img_type = ULONGLONG_IMG; + *fits_datatype = TULONGLONG; + } else if (sizeof(unsigned long) == sizeof(npy_uint64)) { + *fits_img_type = ULONG_IMG; + *fits_datatype = TULONG; + } else if (sizeof(unsigned int) == sizeof(npy_uint64)) { + // there is no UINT_IMG, so use ULONG_IMG + *fits_img_type = ULONG_IMG; + *fits_datatype = TUINT; + } else if (sizeof(unsigned long long) == sizeof(npy_uint64)) { + // we don't expect to get here + *fits_img_type = ULONGLONG_IMG; + *fits_datatype = TULONGLONG; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine 8 byte unsigned integer type"); + *fits_datatype = -9999; + return 1; + } + break; +#else + case NPY_UINT64: + PyErr_SetString(PyExc_TypeError, + "Unsigned 8 byte integer images are not " + "supported by the FITS standard"); + return -9999; +#endif + + case NPY_FLOAT32: + *fits_img_type = FLOAT_IMG; + *fits_datatype = TFLOAT; + break; + case NPY_FLOAT64: + *fits_img_type = DOUBLE_IMG; + *fits_datatype = TDOUBLE; + break; + + default: + sprintf(mess, "Unsupported numpy image datatype %d", npy_dtype); + PyErr_SetString(PyExc_TypeError, mess); + *fits_datatype = -9999; + return 1; + break; + } + + return 0; +} + +/* + * this is really only for reading variable length columns since we should be + * able to just read the bytes for normal columns + */ +static int fits_to_npy_table_type(int fits_dtype, int *isvariable) { + + if (fits_dtype < 0) { + *isvariable = 1; + } else { + *isvariable = 0; + } + + switch (abs(fits_dtype)) { + case TBIT: + return NPY_INT8; + case TLOGICAL: // literal T or F stored as char + return NPY_INT8; + case TBYTE: + return NPY_UINT8; + case TSBYTE: + return NPY_INT8; + + case TUSHORT: + if (sizeof(unsigned short) == sizeof(npy_uint16)) { + return NPY_UINT16; + } else if (sizeof(unsigned short) == sizeof(npy_uint8)) { + return NPY_UINT8; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine numpy type for fits TUSHORT"); + return -9999; + } + case TSHORT: + if (sizeof(short) == sizeof(npy_int16)) { + return NPY_INT16; + } else if (sizeof(short) == sizeof(npy_int8)) { + return NPY_INT8; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine numpy type for fits TSHORT"); + return -9999; + } + + case TUINT: + if (sizeof(unsigned int) == sizeof(npy_uint32)) { + return NPY_UINT32; + } else if (sizeof(unsigned int) == sizeof(npy_uint64)) { + return NPY_UINT64; + } else if (sizeof(unsigned int) == sizeof(npy_uint16)) { + return NPY_UINT16; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine numpy type for fits TUINT"); + return -9999; + } + case TINT: + if (sizeof(int) == sizeof(npy_int32)) { + return NPY_INT32; + } else if (sizeof(int) == sizeof(npy_int64)) { + return NPY_INT64; + } else if (sizeof(int) == sizeof(npy_int16)) { + return NPY_INT16; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine numpy type for fits TINT"); + return -9999; + } + + case TULONG: + if (sizeof(unsigned long) == sizeof(npy_uint32)) { + return NPY_UINT32; + } else if (sizeof(unsigned long) == sizeof(npy_uint64)) { + return NPY_UINT64; + } else if (sizeof(unsigned long) == sizeof(npy_uint16)) { + return NPY_UINT16; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine numpy type for fits TULONG"); + return -9999; + } + case TLONG: + if (sizeof(unsigned long) == sizeof(npy_int32)) { + return NPY_INT32; + } else if (sizeof(unsigned long) == sizeof(npy_int64)) { + return NPY_INT64; + } else if (sizeof(long) == sizeof(npy_int16)) { + return NPY_INT16; + } else { + PyErr_SetString(PyExc_TypeError, + "could not determine numpy type for fits TLONG"); + return -9999; + } + +#ifdef TULONGLONG + case TULONGLONG: + if (sizeof(ULONGLONG) == sizeof(npy_uint64)) { + return NPY_UINT64; + } else if (sizeof(ULONGLONG) == sizeof(npy_uint32)) { + return NPY_UINT32; + } else if (sizeof(ULONGLONG) == sizeof(npy_uint16)) { + return NPY_UINT16; + } else { + PyErr_SetString( + PyExc_TypeError, + "could not determine numpy type for fits TULONGLONG"); + return -9999; + } +#endif + + case TLONGLONG: + if (sizeof(LONGLONG) == sizeof(npy_int64)) { + return NPY_INT64; + } else if (sizeof(LONGLONG) == sizeof(npy_int32)) { + return NPY_INT32; + } else if (sizeof(LONGLONG) == sizeof(npy_int16)) { + return NPY_INT16; + } else { + PyErr_SetString( + PyExc_TypeError, + "could not determine numpy type for fits TLONGLONG"); + return -9999; + } + + case TFLOAT: + return NPY_FLOAT32; + case TDOUBLE: + return NPY_FLOAT64; + + case TCOMPLEX: + return NPY_COMPLEX64; + case TDBLCOMPLEX: + return NPY_COMPLEX128; + + case TSTRING: + return NPY_STRING; + + default: + PyErr_Format(PyExc_TypeError, "Unsupported FITS table datatype %d", + fits_dtype); + return -9999; + } + + return 0; +} + +static int create_empty_hdu(struct PyFITSObject *self) { + int status = 0; + int bitpix = SHORT_IMG; + int naxis = 0; + long *naxes = NULL; + if (fits_create_img(self->fits, bitpix, naxis, naxes, &status)) { + set_ioerr_string_from_status(status, self); + return 1; + } + + return 0; +} + +// follows fits convention that return value is true +// for failure +// +// exception strings are set internally +// +// length checking should happen in python +// +// note tile dims are written reverse order since +// python orders C and fits orders Fortran +static int set_compression(struct PyFITSObject *self, fitsfile *fits, + int comptype, PyObject *tile_dims_obj, int *status) { + + npy_int64 *tile_dims_py = NULL; + long *tile_dims_fits = NULL; + npy_intp ndims = 0, i = 0; + + // can be NOCOMPRESS (0) + if (fits_set_compression_type(fits, comptype, status)) { + set_ioerr_string_from_status(*status, self); + goto _set_compression_bail; + return 1; + } + + if (tile_dims_obj != Py_None) { + + tile_dims_py = + get_int64_from_array((PyArrayObject *)tile_dims_obj, &ndims); + if (tile_dims_py == NULL) { + *status = 1; + } else { + tile_dims_fits = calloc(ndims, sizeof(long)); + if (!tile_dims_fits) { + PyErr_Format(PyExc_MemoryError, "failed to allocate %ld longs", + ndims); + goto _set_compression_bail; + } + + for (i = 0; i < ndims; i++) { + tile_dims_fits[ndims - i - 1] = tile_dims_py[i]; + } + + fits_set_tile_dim(fits, ndims, tile_dims_fits, status); + + free(tile_dims_fits); + tile_dims_fits = NULL; + } + } + +_set_compression_bail: + return *status; +} + +static int pyarray_get_ndim(PyArrayObject *arr) { return arr->nd; } + +/* + Create an image extension, possible writing data as well. + + We allow creating from dimensions rather than from the input image shape, + writing into the HDU later + + It is useful to create the extension first so we can write keywords into the + header before adding data. This avoids moving the data if the header grows + too large. + + However, on distributed file systems it can be more efficient to write + the data at this time due to slowness with updating the file in place. + + */ + +static PyObject *PyFITSObject_create_image_hdu(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int ndims = 0; + long *dims = NULL; + int image_datatype = 0; // fits type for image, AKA bitpix + int datatype = 0; // type for the data we entered + + PyObject *array_obj = NULL, *dims_obj = NULL; + PyArrayObject *array = NULL, *dims_array = NULL; + PyObject *comptype_obj = NULL; + PyObject *tile_dims_obj = NULL; + PyObject *qlevel_obj = NULL; + PyObject *qmethod_obj = NULL; + PyObject *dither_seed_obj = NULL; + PyObject *hcomp_scale_obj = NULL; + PyObject *hcomp_smooth_obj = NULL; + + int npy_dtype = 0, nkeys = 0, write_data = 0; + int i = 0; + int status = 0; + int py_status = 0; + + char *extname = NULL; + int extver = 0; + float qlevel = 0; + int qmethod = 0; + int dither_seed = 0; + float hcomp_scale = 0; + int hcomp_smooth = 0; + int comptype = 0; + int orig_dither_seed; + int got_dither_seed = 0; + int any_nan = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + // We do allow overriding the dither seed and then putting it back + // Luckily cfitsio has public APIs for that so this should be robust + if (fits_get_dither_seed(self->fits, &orig_dither_seed, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } else { + // tell the code to put back the dither seed later + got_dither_seed = 1; + } + + static char *kwlist[] = { + "array", "nkeys", "dims", "comptype", "tile_dims", + "qlevel", "qmethod", "dither_seed", "hcomp_scale", "hcomp_smooth", + "extname", "extver", "any_nan", NULL, + }; + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "Oi|OOOOOOOOsii", kwlist, &array_obj, &nkeys, &dims_obj, + &comptype_obj, &tile_dims_obj, &qlevel_obj, &qmethod_obj, + &dither_seed_obj, &hcomp_scale_obj, &hcomp_smooth_obj, &extname, + &extver, &any_nan)) { + py_status = 1; + goto create_image_hdu_cleanup; + } + + if (array_obj == Py_None) { + if (create_empty_hdu(self)) { + // error string is set in create_empty_hdu + return NULL; + } + } else { + array = (PyArrayObject *)array_obj; + if (!PyArray_Check(array)) { + PyErr_SetString(PyExc_TypeError, "input must be an array."); + py_status = 1; + goto create_image_hdu_cleanup; + } + + npy_dtype = PyArray_TYPE(array); + if (npy_to_fits_image_types(npy_dtype, &image_datatype, &datatype)) { + py_status = 1; + goto create_image_hdu_cleanup; + } + + if (PyArray_Check(dims_obj)) { + // get dims from input, which must be of type 'i8' + // this means we are not writing the array that was input, + // it is only used to determine the data type + dims_array = (PyArrayObject *)dims_obj; + + npy_int64 *tptr = NULL, tmp = 0; + ndims = PyArray_SIZE(dims_array); + dims = calloc(ndims, sizeof(long)); + for (i = 0; i < ndims; i++) { + tptr = (npy_int64 *)PyArray_GETPTR1(dims_array, i); + tmp = *tptr; + dims[ndims - i - 1] = (long)tmp; + } + write_data = 0; + } else { + // we get the dimensions from the array, which means we are going + // to write it as well + ndims = pyarray_get_ndim(array); + dims = calloc(ndims, sizeof(long)); + for (i = 0; i < ndims; i++) { + dims[ndims - i - 1] = PyArray_DIM(array, i); + } + write_data = 1; + } + + if (comptype_obj != Py_None) { + comptype = (int)PyLong_AsLong(comptype_obj); + } else { + // get what the code is going to use just in case we override the + // HCOMPRESS options + comptype = self->fits->Fptr->request_compress_type; + } + + if (qlevel_obj != Py_None) { + qlevel = (float)PyFloat_AsDouble(qlevel_obj); + } + + if (qmethod_obj != Py_None) { + qmethod = (int)PyLong_AsLong(qmethod_obj); + } + + if (dither_seed_obj != Py_None) { + dither_seed = (int)PyLong_AsLong(dither_seed_obj); + } + + if (hcomp_scale_obj != Py_None) { + hcomp_scale = (float)PyFloat_AsDouble(hcomp_scale_obj); + } + + if (hcomp_smooth_obj != Py_None) { + hcomp_smooth = (int)PyLong_AsLong(hcomp_smooth_obj); + } + + if ((comptype_obj != Py_None) || (tile_dims_obj != Py_None)) { + // exception strings are set internally + if (set_compression(self, self->fits, comptype, tile_dims_obj, + &status)) { + goto create_image_hdu_cleanup; + } + } + + if (qlevel_obj != Py_None) { + if (fits_set_quantize_level(self->fits, qlevel, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + + if (qmethod_obj != Py_None) { + if (fits_set_quantize_method(self->fits, qmethod, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + + if (dither_seed_obj != Py_None) { + if (fits_set_dither_seed(self->fits, dither_seed, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + + if (comptype == HCOMPRESS_1) { + if (hcomp_scale_obj != Py_None) { + if (fits_set_hcomp_scale(self->fits, hcomp_scale, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + + if (hcomp_smooth_obj != Py_None) { + if (fits_set_hcomp_smooth(self->fits, hcomp_smooth, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + } + + if (fits_create_img(self->fits, image_datatype, ndims, dims, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + if (extname != NULL) { + if (strlen(extname) > 0) { + + // comments are NULL + if (fits_update_key_str(self->fits, "EXTNAME", extname, NULL, + &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + if (extver > 0) { + if (fits_update_key_lng(self->fits, "EXTVER", (LONGLONG)extver, + NULL, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + } + } + + if (nkeys > 0) { + if (fits_set_hdrsize(self->fits, nkeys, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + + if (write_data) { + int firstpixel = 1; + LONGLONG nelements = 0; + void *data = NULL; + nelements = PyArray_SIZE(array); + data = PyArray_DATA(array); + if (any_nan) { + float fnullval = INFINITY; + double dnullval = INFINITY; + void *nullval_ptr; + + if (datatype == TFLOAT) { + nullval_ptr = (void *)(&fnullval); + } else if (datatype == TDOUBLE) { + nullval_ptr = (void *)(&dnullval); + } else { + nullval_ptr = NULL; + } + + if (fits_write_imgnull(self->fits, datatype, firstpixel, nelements, + data, nullval_ptr, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } else { + if (fits_write_img(self->fits, datatype, firstpixel, nelements, + data, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + } + } + + // this flushes all buffers + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + goto create_image_hdu_cleanup; + } + +create_image_hdu_cleanup: + free(dims); + dims = NULL; + if ((comptype_obj != Py_None) || (tile_dims_obj != Py_None) || + (qlevel_obj != Py_None) || (qmethod_obj != Py_None) || + (hcomp_scale_obj != Py_None) || (hcomp_smooth_obj != Py_None)) { + if (fits_unset_compression_request(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + } + } + + // put back the dither seed after we reset any compression params + if (got_dither_seed == 1) { + if (fits_set_dither_seed(self->fits, orig_dither_seed, &status)) { + set_ioerr_string_from_status(status, self); + } + } + + if (status != 0 || py_status != 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +// reshape the image to specified dims +// the input array must be of type int64 +static PyObject *PyFITSObject_reshape_image(struct PyFITSObject *self, + PyObject *args) { + + int status = 0; + int hdunum = 0, hdutype = 0; + PyObject *dims_obj = NULL; + PyArrayObject *dims_array = NULL; + LONGLONG dims[CFITSIO_MAX_ARRAY_DIMS] = {0}; + LONGLONG dims_orig[CFITSIO_MAX_ARRAY_DIMS] = {0}; + int ndims = 0, ndims_orig = 0; + npy_int64 dim = 0; + npy_intp i = 0; + int bitpix = 0, maxdim = CFITSIO_MAX_ARRAY_DIMS; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"iO", &hdunum, &dims_obj)) { + return NULL; + } + dims_array = (PyArrayObject *)dims_obj; + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // existing image params, just to get bitpix + if (fits_get_img_paramll(self->fits, maxdim, &bitpix, &ndims_orig, + dims_orig, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + ndims = PyArray_SIZE(dims_array); + for (i = 0; i < ndims; i++) { + dim = *(npy_int64 *)PyArray_GETPTR1(dims_array, i); + dims[i] = (LONGLONG)dim; + } + + if (fits_resize_imgll(self->fits, bitpix, ndims, dims, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// write the image to an existing HDU created using create_image_hdu +// dims are not checked +static PyObject *PyFITSObject_write_image(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + LONGLONG nelements = 1; + PY_LONG_LONG firstpixel_py = 0; + LONGLONG firstpixel = 0; + PY_LONG_LONG any_nan_py = 0; + LONGLONG any_nan = 0; + int image_datatype = 0; // fits type for image, AKA bitpix + int datatype = 0; // type for the data we entered + + PyObject *array_obj = NULL; + PyArrayObject *array = NULL; + void *data = NULL; + int npy_dtype = 0; + int status = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"iOLL", &hdunum, &array_obj, + &firstpixel_py, &any_nan_py)) { + return NULL; + } + array = (PyArrayObject *)array_obj; + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (!PyArray_Check(array)) { + PyErr_SetString(PyExc_TypeError, "input must be an array."); + return NULL; + } + + npy_dtype = PyArray_TYPE(array); + if (npy_to_fits_image_types(npy_dtype, &image_datatype, &datatype)) { + return NULL; + } + + data = PyArray_DATA(array); + nelements = PyArray_SIZE(array); + firstpixel = (LONGLONG)firstpixel_py; + any_nan = (LONGLONG)any_nan_py; + + if (any_nan_py) { + float fnullval = INFINITY; + double dnullval = INFINITY; + void *nullval_ptr; + + if (datatype == TFLOAT) { + nullval_ptr = (void *)(&fnullval); + } else if (datatype == TDOUBLE) { + nullval_ptr = (void *)(&dnullval); + } else { + nullval_ptr = NULL; + } + + if (fits_write_imgnull(self->fits, datatype, firstpixel, nelements, + data, nullval_ptr, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + } else { + if (fits_write_img(self->fits, datatype, firstpixel, nelements, data, + &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + } + // this is a full file close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// write a rectangular subset to the image in an existing HDU +// created using create_image_hdu +// dims are not checked +static PyObject *PyFITSObject_write_subset(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + int image_datatype = 0; // fits type for image, AKA bitpix + int datatype = 0; // type for the data we entered + + long fpixel[CFITSIO_MAX_ARRAY_DIMS] = {0}; + long lpixel[CFITSIO_MAX_ARRAY_DIMS] = {0}; + npy_int64 pixval = 0; + npy_intp i = 0; + + PyObject *array_obj = NULL; + PyObject *firstpixel_obj = NULL; + PyObject *lastpixel_obj = NULL; + PyArrayObject *array = NULL; + PyArrayObject *firstpixel = NULL; + PyArrayObject *lastpixel = NULL; + void *data = NULL; + int npy_dtype = 0, ndims; + int status = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"iOOO", &hdunum, &array_obj, + &firstpixel_obj, &lastpixel_obj)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + array = (PyArrayObject *)array_obj; + if (!PyArray_Check(array)) { + PyErr_SetString(PyExc_TypeError, "input data must be an array."); + return NULL; + } + npy_dtype = PyArray_TYPE(array); + if (npy_to_fits_image_types(npy_dtype, &image_datatype, &datatype)) { + return NULL; + } + + firstpixel = (PyArrayObject *)firstpixel_obj; + if (!PyArray_Check(firstpixel)) { + PyErr_SetString(PyExc_TypeError, "input firstpixel must be an array."); + return NULL; + } + + lastpixel = (PyArrayObject *)lastpixel_obj; + if (!PyArray_Check(lastpixel)) { + PyErr_SetString(PyExc_TypeError, "input lastpixel must be an array."); + return NULL; + } + + if (PyArray_SIZE(firstpixel) != PyArray_SIZE(lastpixel)) { + PyErr_SetString( + PyExc_TypeError, + "input firstpixel and lastpixel must have the same size."); + return NULL; + } + + ndims = PyArray_SIZE(firstpixel); + for (i = 0; i < ndims; i++) { + pixval = *(npy_int64 *)PyArray_GETPTR1(firstpixel, i); + fpixel[i] = (long)pixval; + + pixval = *(npy_int64 *)PyArray_GETPTR1(lastpixel, i); + lpixel[i] = (long)pixval; + } + + data = PyArray_DATA(array); + if (fits_write_subset(self->fits, datatype, fpixel, lpixel, data, + &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this is a full file close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +/* + * Write tdims from the list. The list must be the expected length. + * Entries must be strings or None; if None the tdim is not written. + * + * The keys are written as TDIM{colnum} + */ +static int add_tdims_from_listobj(struct PyFITSObject *self, fitsfile *fits, + PyObject *tdimObj, int ncols) { + int status = 0, i = 0; + size_t size = 0; + char keyname[20]; + int colnum = 0; + PyObject *tmp = NULL; + char *tdim = NULL; + + if (tdimObj == NULL || tdimObj == Py_None) { + // it is ok for it to be empty + return 0; + } + + if (!PyList_Check(tdimObj)) { + PyErr_SetString(PyExc_ValueError, "Expected a list for tdims"); + return 1; + } + + size = PyList_Size(tdimObj); + if (size != (size_t)ncols) { + PyErr_Format(PyExc_ValueError, + "Expected %d elements in tdims list, got %ld", ncols, + size); + return 1; + } + + for (i = 0; i < ncols; i++) { + colnum = i + 1; + tmp = PyList_GetItem(tdimObj, i); + if (tmp != Py_None) { + if (!is_python_string(tmp)) { + PyErr_SetString(PyExc_ValueError, + "Expected only strings or None for tdim"); + return 1; + } + + sprintf(keyname, "TDIM%d", colnum); + + tdim = get_object_as_string(tmp); + fits_write_key(fits, TSTRING, keyname, tdim, NULL, &status); + free(tdim); + + if (status) { + set_ioerr_string_from_status(status, self); + return 1; + } + } + } + + return 0; +} + +// create a new table structure. No physical rows are added yet. +static PyObject *PyFITSObject_create_table_hdu(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int table_type = 0, nkeys = 0; + int nfields = 0; + LONGLONG nrows = 0; // start empty + + static char *kwlist[] = {"table_type", "nkeys", "ttyp", + "tform", "tunit", "tdim", + "extname", "extver", NULL}; + // these are all strings + PyObject *ttypObj = NULL; + PyObject *tformObj = NULL; + PyObject *tunitObj = NULL; // optional + PyObject *tdimObj = NULL; // optional + + // these must be freed + struct stringlist *ttyp = NULL; + struct stringlist *tform = NULL; + struct stringlist *tunit = NULL; + // struct stringlist* tdim=stringlist_new(); + char *extname = NULL; + char *extname_use = NULL; + int extver = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iiOO|OOsi", kwlist, + &table_type, &nkeys, &ttypObj, &tformObj, + &tunitObj, &tdimObj, &extname, &extver)) { + return NULL; + } + + ttyp = stringlist_new(); + tform = stringlist_new(); + tunit = stringlist_new(); + if (stringlist_addfrom_listobj(ttyp, ttypObj, "names")) { + status = 99; + goto create_table_cleanup; + } + + if (stringlist_addfrom_listobj(tform, tformObj, "formats")) { + status = 99; + goto create_table_cleanup; + } + + if (tunitObj != NULL && tunitObj != Py_None) { + if (stringlist_addfrom_listobj(tunit, tunitObj, "units")) { + status = 99; + goto create_table_cleanup; + } + } + + if (extname != NULL) { + if (strlen(extname) > 0) { + extname_use = extname; + } + } + nfields = ttyp->size; + if (fits_create_tbl(self->fits, table_type, nrows, nfields, ttyp->data, + tform->data, tunit->data, extname_use, &status)) { + set_ioerr_string_from_status(status, self); + goto create_table_cleanup; + } + + if (add_tdims_from_listobj(self, self->fits, tdimObj, nfields)) { + status = 99; + goto create_table_cleanup; + } + + if (extname_use != NULL) { + if (extver > 0) { + + if (fits_update_key_lng(self->fits, "EXTVER", (LONGLONG)extver, + NULL, &status)) { + set_ioerr_string_from_status(status, self); + goto create_table_cleanup; + } + } + } + + if (nkeys > 0) { + if (fits_set_hdrsize(self->fits, nkeys, &status)) { + set_ioerr_string_from_status(status, self); + goto create_table_cleanup; + } + } + + // this does a full close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + goto create_table_cleanup; + } + +create_table_cleanup: + ttyp = stringlist_delete(ttyp); + tform = stringlist_delete(tform); + tunit = stringlist_delete(tunit); + // tdim = stringlist_delete(tdim); + + if (status != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +// create a new table structure. No physical rows are added yet. +static PyObject *PyFITSObject_insert_col(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int hdunum = 0; + int colnum = 0; + + int hdutype = 0; + + static char *kwlist[] = {"hdunum", "colnum", "ttyp", "tform", "tdim", NULL}; + // these are all strings + char *ttype = NULL; // field name + char *tform = NULL; // format + PyObject *tdimObj = NULL; // optional, a list of len 1 + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iiss|O", kwlist, &hdunum, + &colnum, &ttype, &tform, &tdimObj)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_insert_col(self->fits, colnum, ttype, tform, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // OK if dims are not sent + if (tdimObj != NULL && tdimObj != Py_None) { + PyObject *tmp = NULL; + char *tdim = NULL; + char keyname[20]; + + sprintf(keyname, "TDIM%d", colnum); + tmp = PyList_GetItem(tdimObj, 0); + + tdim = get_object_as_string(tmp); + fits_write_key(self->fits, TSTRING, keyname, tdim, NULL, &status); + free(tdim); + + if (status) { + set_ioerr_string_from_status(status, self); + return NULL; + } + } + + // this does a full close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// No error checking performed here +static int write_string_column( + struct PyFITSObject *self, fitsfile *fits, /* I - FITS file pointer */ + int hdutype, /* I - one of ASCII_TABLE or BINARY_TABLE */ + int colnum, /* I - number of column to write (1 = 1st col) */ + LONGLONG firstrow, /* I - first row to write (1 = 1st row) */ + LONGLONG firstelem, /* I - first vector element to write (1 = 1st) */ + LONGLONG nelem, /* I - number of strings to write */ + char *data, int *status) { /* IO - error status */ + + if (hdutype == ASCII_TBL || CFITSIO_MAJOR < 4) { + LONGLONG i = 0; + LONGLONG twidth = 0; + // need to create a char** representation of the data, just point back + // into the data array at string width offsets. the fits_write_col_str + // takes care of skipping between fields. + char *cdata = NULL; + char **strdata = NULL; + + // using struct def here, could cause problems + twidth = fits->Fptr->tableptr[colnum - 1].twidth; + + strdata = malloc(nelem * sizeof(char *)); + if (strdata == NULL) { + PyErr_SetString(PyExc_MemoryError, + "could not allocate temporary string pointers"); + *status = 99; + return 1; + } + cdata = (char *)data; + for (i = 0; i < nelem; i++) { + strdata[i] = &cdata[twidth * i]; + } + + if (fits_write_col_str(fits, colnum, firstrow, firstelem, nelem, + strdata, status)) { + set_ioerr_string_from_status(*status, self); + free(strdata); + return 1; + } + + free(strdata); + } else { // BINARY_TABLE + // for string columns in binary tables, cfitsio has a special mode + // where you can write the the bytes directly. let's do that to + // preserve nulls and prevent + // cfitsio from padding with spaces + + LONGLONG nelem_byt; + unsigned char *strdata_byt; + + // using struct def here, could cause problems + nelem_byt = nelem * (fits->Fptr->tableptr[colnum - 1].twidth); + strdata_byt = (unsigned char *)data; + + if (fits_write_col_byt(fits, colnum, firstrow, firstelem, nelem_byt, + strdata_byt, status)) { + set_ioerr_string_from_status(*status, self); + return 1; + } + } + + return 0; +} + +// write a column, starting at firstrow. On the python side, the firstrow kwd +// should default to 1. +// You can append rows using firstrow = nrows+1 +/* +static PyObject * +PyFITSObject_write_column(struct PyFITSObject* self, PyObject* args, PyObject* +kwds) { int status=0; int hdunum=0; int hdutype=0; int colnum=0; int +write_bitcols=0; PyObject* array=NULL; + + void* data=NULL; + PY_LONG_LONG firstrow_py=0; + LONGLONG firstrow=1; + LONGLONG firstelem=1; + LONGLONG nelem=0; + int npy_dtype=0; + int fits_dtype=0; + + static char *kwlist[] = +{"hdunum","colnum","array","firstrow","write_bitcols", NULL}; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iiOLi", + kwlist, &hdunum, &colnum, &array, +&firstrow_py, &write_bitcols)) { return NULL; + } + firstrow = (LONGLONG) firstrow_py; + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + + if (!PyArray_Check(array)) { + PyErr_SetString(PyExc_ValueError,"only arrays can be written to +columns"); return NULL; + } + + npy_dtype = PyArray_TYPE(array); + fits_dtype = npy_to_fits_table_type(npy_dtype, write_bitcols); + if (fits_dtype == -9999) { + return NULL; + } + if (fits_dtype == TLOGICAL) { + int tstatus=0, ttype=0; + LONGLONG trepeat=0, twidth=0; + // if the column exists and is declared TBIT we will write + // that way instead + if (fits_get_coltypell(self->fits, colnum, + &ttype, &trepeat, &twidth, &tstatus)==0) { + // if we don't get here its because the column doesn't exist + // yet and that's ok + if (ttype==TBIT) { + fits_dtype=TBIT; + } + } + } + + + + data = PyArray_DATA(array); + nelem = PyArray_SIZE(array); + + if (fits_dtype == TSTRING) { + + // this is my wrapper for strings + if (write_string_column(self, self->fits, colnum, firstrow, firstelem, +nelem, data, &status)) { set_ioerr_string_from_status(status, self); return +NULL; + } + } else if (fits_dtype == TBIT) { + if (fits_write_col_bit(self->fits, colnum, firstrow, firstelem, nelem, +data, &status)) { set_ioerr_string_from_status(status, self); return NULL; + } + } else { + if( fits_write_col(self->fits, fits_dtype, colnum, firstrow, firstelem, +nelem, data, &status)) { set_ioerr_string_from_status(status, self); return +NULL; + } + } + + // this is a full file close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + + Py_RETURN_NONE; +} +*/ + +static PyObject *PyFITSObject_write_columns(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + int write_bitcols = 0; + // void **data_ptrs=NULL; + PyObject *colnum_list = NULL; + PyObject *array_list = NULL; + PyObject *tmp_obj = NULL; + PyArrayObject *tmp_array = NULL; + + Py_ssize_t ncols = 0; + + void *data = NULL; + PY_LONG_LONG firstrow_py = 0; + LONGLONG firstrow = 1, thisrow = 0; + LONGLONG firstelem = 1; + LONGLONG nelem = 0; + LONGLONG *nperrow = NULL; + int npy_dtype = 0; + int *fits_dtypes = NULL; + int *is_string = NULL, *colnums = NULL; + void **array_ptrs = NULL; + + npy_intp ndim = 0, *dims = NULL; + Py_ssize_t irow = 0, icol = 0, j = 0; + ; + + static char *kwlist[] = {"hdunum", "colnums", "arraylist", + "firstrow", "write_bitcols", NULL}; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iOOLi", kwlist, &hdunum, + &colnum_list, &array_list, &firstrow_py, + &write_bitcols)) { + return NULL; + } + firstrow = (LONGLONG)firstrow_py; + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (!PyList_Check(colnum_list)) { + PyErr_SetString(PyExc_ValueError, "colnums must be a list"); + return NULL; + } + if (!PyList_Check(array_list)) { + PyErr_SetString(PyExc_ValueError, "colnums must be a list"); + return NULL; + } + ncols = PyList_Size(colnum_list); + if (ncols == 0) { + goto _fitsio_pywrap_write_columns_bail; + } + if (ncols != PyList_Size(array_list)) { + PyErr_Format(PyExc_ValueError, + "colnum and array lists not same size: %ld/%ld", ncols, + PyList_Size(array_list)); + } + + // from here on we'll have some temporary arrays we have to free + is_string = calloc(ncols, sizeof(int)); + colnums = calloc(ncols, sizeof(int)); + array_ptrs = calloc(ncols, sizeof(void *)); + nperrow = calloc(ncols, sizeof(LONGLONG)); + fits_dtypes = calloc(ncols, sizeof(int)); + + for (icol = 0; icol < ncols; icol++) { + + tmp_obj = PyList_GetItem(colnum_list, icol); +#if PY_MAJOR_VERSION >= 3 + colnums[icol] = 1 + (int)PyLong_AsLong(tmp_obj); +#else + colnums[icol] = 1 + (int)PyInt_AsLong(tmp_obj); +#endif + + tmp_array = (PyArrayObject *)PyList_GetItem(array_list, icol); + npy_dtype = PyArray_TYPE(tmp_array); + + fits_dtypes[icol] = npy_to_fits_table_type(npy_dtype, write_bitcols); + if (fits_dtypes[icol] == -9999) { + status = 1; + goto _fitsio_pywrap_write_columns_bail; + } + if (fits_dtypes[icol] == TLOGICAL) { + int tstatus = 0, ttype = 0; + LONGLONG trepeat = 0, twidth = 0; + // if the column exists and is declared TBIT we will write + // that way instead + if (fits_get_coltypell(self->fits, colnums[icol], &ttype, &trepeat, + &twidth, &tstatus) == 0) { + // if we don't get here its because the column doesn't exist + // yet and that's ok + if (ttype == TBIT) { + fits_dtypes[icol] = TBIT; + } + } + } + + if (fits_dtypes[icol] == TSTRING) { + is_string[icol] = 1; + } + ndim = PyArray_NDIM(tmp_array); + dims = PyArray_DIMS(tmp_array); + if (icol == 0) { + nelem = dims[0]; + } else { + if (dims[0] != nelem) { + PyErr_Format(PyExc_ValueError, + "not all entries have same row count, " + "%lld/%ld", + nelem, dims[0]); + status = 1; + goto _fitsio_pywrap_write_columns_bail; + } + } + + array_ptrs[icol] = tmp_array; + + nperrow[icol] = 1; + for (j = 1; j < ndim; j++) { + nperrow[icol] *= dims[j]; + } + } + + for (irow = 0; irow < nelem; irow++) { + thisrow = firstrow + irow; + for (icol = 0; icol < ncols; icol++) { + data = PyArray_GETPTR1(array_ptrs[icol], irow); + if (is_string[icol]) { + if (write_string_column(self, self->fits, hdutype, + colnums[icol], thisrow, firstelem, + nperrow[icol], (char *)data, &status)) { + set_ioerr_string_from_status(status, self); + goto _fitsio_pywrap_write_columns_bail; + } + + } else if (fits_dtypes[icol] == TBIT) { + if (fits_write_col_bit(self->fits, colnums[icol], thisrow, + firstelem, nperrow[icol], data, + &status)) { + set_ioerr_string_from_status(status, self); + goto _fitsio_pywrap_write_columns_bail; + } + } else { + if (fits_write_col(self->fits, fits_dtypes[icol], colnums[icol], + thisrow, firstelem, nperrow[icol], data, + &status)) { + set_ioerr_string_from_status(status, self); + goto _fitsio_pywrap_write_columns_bail; + } + } + } + } + /* + nelem = PyArray_SIZE(array); + + if (fits_dtype == TSTRING) { + + // this is my wrapper for strings + if (write_string_column(self, self->fits, colnum, firstrow, firstelem, + nelem, data, &status)) { set_ioerr_string_from_status(status, self); return + NULL; + } + + } else { + if( fits_write_col(self->fits, fits_dtype, colnum, firstrow, firstelem, + nelem, data, &status)) { set_ioerr_string_from_status(status, self); return + NULL; + } + } + + // this is a full file close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + */ + +_fitsio_pywrap_write_columns_bail: + free(is_string); + is_string = NULL; + free(colnums); + colnums = NULL; + free(array_ptrs); + array_ptrs = NULL; + free(nperrow); + nperrow = NULL; + free(fits_dtypes); + fits_dtypes = NULL; + if (status != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +// No error checking performed here +static int write_var_string_column( + fitsfile *fits, /* I - FITS file pointer */ + int colnum, /* I - number of column to write (1 = 1st col) */ + LONGLONG firstrow, /* I - first row to write (1 = 1st row) */ + PyArrayObject *array, int *status) { /* IO - error status */ + + LONGLONG firstelem = 1; // ignored + LONGLONG nelem = 1; // ignored + npy_intp nrows = 0; + npy_intp i = 0; + char *ptr = NULL; + int res = 0; + + PyObject *el = NULL; + char *strdata = NULL; + char *strarr[1]; + + nrows = PyArray_SIZE(array); + for (i = 0; i < nrows; i++) { + ptr = PyArray_GetPtr(array, &i); + el = PyArray_GETITEM(array, ptr); + + strdata = get_object_as_string(el); + + // just a container + strarr[0] = strdata; + res = fits_write_col_str(fits, colnum, firstrow + i, firstelem, nelem, + strarr, status); + + free(strdata); + if (res > 0) { + goto write_var_string_column_cleanup; + } + } + +write_var_string_column_cleanup: + + if (*status > 0) { + return 1; + } + + return 0; +} + +/* + * No error checking performed here + */ +static int write_var_num_column( + struct PyFITSObject *self, fitsfile *fits, /* I - FITS file pointer */ + int colnum, /* I - number of column to write (1 = 1st col) */ + LONGLONG firstrow, /* I - first row to write (1 = 1st row) */ + int fits_dtype, PyArrayObject *array, int *status) { /* IO - error status */ + + LONGLONG firstelem = 1; + npy_intp nelem = 0; + npy_intp nrows = 0; + npy_intp i = 0; + PyObject *el = NULL; + PyArrayObject *el_array = NULL; + void *data = NULL; + void *ptr = NULL; + + int npy_dtype = 0, isvariable = 0; + + int mindepth = 1, maxdepth = 0; + PyObject *context = NULL; + int requirements = NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_ALIGNED | + NPY_ARRAY_NOTSWAPPED | NPY_ARRAY_ELEMENTSTRIDES; + + int res = 0; + + npy_dtype = fits_to_npy_table_type(fits_dtype, &isvariable); + + nrows = PyArray_SIZE(array); + for (i = 0; i < nrows; i++) { + ptr = PyArray_GetPtr((PyArrayObject *)array, &i); + el = PyArray_GETITEM(array, ptr); + + // a copy is only made if needed + el_array = (PyArrayObject *)PyArray_CheckFromAny( + el, PyArray_DescrFromType(npy_dtype), mindepth, maxdepth, + requirements, context); + if (el_array == NULL) { + // error message will already be set + return 1; + } + + nelem = PyArray_SIZE(el_array); + data = PyArray_DATA(el_array); + res = fits_write_col(fits, abs(fits_dtype), colnum, firstrow + i, + firstelem, (LONGLONG)nelem, data, status); + Py_XDECREF(el_array); + + if (res > 0) { + set_ioerr_string_from_status(*status, self); + return 1; + } + } + + return 0; +} + +/* + * write a variable length column, starting at firstrow. On the python side, + * the firstrow kwd should default to 1. You can append rows using firstrow = + * nrows+1 + * + * The input array should be of type NPY_OBJECT, and the elements + * should be either all strings or numpy arrays of the same type + */ + +static PyObject *PyFITSObject_write_var_column(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + int colnum = 0; + PyObject *array_obj = NULL; + PyArrayObject *array = NULL; + + PY_LONG_LONG firstrow_py = 0; + LONGLONG firstrow = 1; + int npy_dtype = 0; + int fits_dtype = 0; + + static char *kwlist[] = {"hdunum", "colnum", "array", "firstrow", NULL}; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iiOL", kwlist, &hdunum, + &colnum, &array_obj, &firstrow_py)) { + return NULL; + } + firstrow = (LONGLONG)firstrow_py; + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (!PyArray_Check(array_obj)) { + PyErr_SetString(PyExc_ValueError, + "only arrays can be written to columns"); + return NULL; + } + array = (PyArrayObject *)array_obj; + + npy_dtype = PyArray_TYPE(array); + if (npy_dtype != NPY_OBJECT) { + PyErr_SetString( + PyExc_TypeError, + "only object arrays can be written to variable length columns"); + return NULL; + } + + // determine the fits dtype for this column. We will use this to get data + // from the array for writing + if (fits_get_eqcoltypell(self->fits, colnum, &fits_dtype, NULL, NULL, + &status) > 0) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_dtype == -TSTRING) { + if (write_var_string_column(self->fits, colnum, firstrow, array, + &status)) { + if (status != 0) { + set_ioerr_string_from_status(status, self); + } + return NULL; + } + } else { + if (write_var_num_column(self, self->fits, colnum, firstrow, fits_dtype, + array, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + } + + // this is a full file close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +/* + case for writing an entire record +*/ +static PyObject *PyFITSObject_write_record(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *cardin = NULL; + char card[FLEN_CARD]; + + if (!PyArg_ParseTuple(args, (char *)"is", &hdunum, &cardin)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + // card not null terminated, so we copy everything. GCC will + // warn about this + strncpy(card, cardin, FLEN_CARD); + + if (fits_write_record(self->fits, card, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// let python do the conversions +static PyObject *PyFITSObject_write_string_key(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *keyname = NULL; + char *value = NULL; + char *comment = NULL; + char *comment_in = NULL; + + if (!PyArg_ParseTuple(args, (char *)"isss", &hdunum, &keyname, &value, + &comment_in)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (strlen(comment_in) > 0) { + comment = comment_in; + } + + if (fits_update_key_longstr(self->fits, keyname, value, comment, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject *PyFITSObject_write_double_key(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + int decimals = -15; + + char *keyname = NULL; + double value = 0; + char *comment = NULL; + char *comment_in = NULL; + + if (!PyArg_ParseTuple(args, (char *)"isds", &hdunum, &keyname, &value, + &comment_in)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (strlen(comment_in) > 0) { + comment = comment_in; + } + + if (fits_update_key_dbl(self->fits, keyname, value, decimals, comment, + &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject *PyFITSObject_write_long_long_key(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *keyname = NULL; + long long value = 0; + char *comment = NULL; + char *comment_in = NULL; + + if (!PyArg_ParseTuple(args, (char *)"isLs", &hdunum, &keyname, &value, + &comment_in)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (strlen(comment_in) > 0) { + comment = comment_in; + } + + if (fits_update_key_lng(self->fits, keyname, (LONGLONG)value, comment, + &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject *PyFITSObject_write_logical_key(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *keyname = NULL; + int value = 0; + char *comment = NULL; + char *comment_in = NULL; + + if (!PyArg_ParseTuple(args, (char *)"isis", &hdunum, &keyname, &value, + &comment_in)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (strlen(comment_in) > 0) { + comment = comment_in; + } + + if (fits_update_key_log(self->fits, keyname, value, comment, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// let python do the conversions +static PyObject *PyFITSObject_write_comment(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *comment = NULL; + + if (!PyArg_ParseTuple(args, (char *)"is", &hdunum, &comment)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_write_comment(self->fits, comment, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// let python do the conversions +static PyObject *PyFITSObject_write_history(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *history = NULL; + + if (!PyArg_ParseTuple(args, (char *)"is", &hdunum, &history)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_write_history(self->fits, history, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// ADW: Adapted from ffpcom and ffphis in putkey.c +static int fits_write_continue(fitsfile *fptr, /* I - FITS file pointer */ + const char *cont, /* I - continue string */ + int *status) /* IO - error status */ +/* + Write 1 or more CONTINUE keywords. If the history string is too + long to fit on a single keyword (72 chars) then it will automatically + be continued on multiple CONTINUE keywords. +*/ +{ + char card[FLEN_CARD]; + int len, ii; + + if (*status > 0) /* inherit input status value if > 0 */ + return (*status); + + len = strlen(cont); + ii = 0; + + for (; len > 0; len -= 72) { + strcpy(card, "CONTINUE"); + strncat(card, &cont[ii], 72); + ffprec(fptr, card, status); + ii += 72; + } + + return (*status); +} + +// let python do the conversions +static PyObject *PyFITSObject_write_continue(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *value = NULL; + + if (!PyArg_ParseTuple(args, (char *)"is", &hdunum, &value)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_write_continue(self->fits, value, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject *PyFITSObject_write_undefined_key(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *keyname = NULL; + char *comment = NULL; + char *comment_in = NULL; + + if (!PyArg_ParseTuple(args, (char *)"iss", &hdunum, &keyname, + &comment_in)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (strlen(comment_in) > 0) { + comment = comment_in; + } + + if (fits_update_key_null(self->fits, keyname, comment, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +// let python do the conversions +static PyObject *PyFITSObject_delete_key(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + char *keyname = NULL; + + if (!PyArg_ParseTuple(args, (char *)"is", &hdunum, &keyname)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_delete_key(self->fits, keyname, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does not close and reopen + if (fits_flush_buffer(self->fits, 0, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +/* + insert a set of rows +*/ + +static PyObject *PyFITSObject_insert_rows(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int hdunum = 0; + + int hdutype = 0; + PY_LONG_LONG firstrow_py = 0, nrows_py = 0; + LONGLONG firstrow = 0, nrows = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"iLL", &hdunum, &firstrow_py, + &nrows_py)) { + return NULL; + } + + firstrow = (LONGLONG)firstrow_py; + nrows = (LONGLONG)nrows_py; + + if (nrows <= 0) { + // nothing to do, just return + Py_RETURN_NONE; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_insert_rows(self->fits, firstrow, nrows, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does a full close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +/* + + delete a range of rows + + input stop is like a python slice, so exclusive, but 1-offset + rather than 0-offset +*/ + +static PyObject *PyFITSObject_delete_row_range(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int hdunum = 0; + + int hdutype = 0; + PY_LONG_LONG slice_start_py = 0, slice_stop_py = 0; + LONGLONG slice_start = 0, slice_stop = 0, nrows = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"iLL", &hdunum, &slice_start_py, + &slice_stop_py)) { + return NULL; + } + + slice_start = (LONGLONG)slice_start_py; + slice_stop = (LONGLONG)slice_stop_py; + nrows = slice_stop - slice_start; + + if (nrows <= 0) { + // nothing to do, just return + Py_RETURN_NONE; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_delete_rows(self->fits, slice_start, nrows, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does a full close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +/* + + delete a specific set of rows, 1-offset + + no type checking is applied to the rows +*/ + +static PyObject *PyFITSObject_delete_rows(struct PyFITSObject *self, + PyObject *args, PyObject *kwds) { + int status = 0; + int hdunum = 0; + + int hdutype = 0; + PyObject *rows_obj = NULL; + PyArrayObject *rows_array = NULL; + LONGLONG *rows = NULL, nrows = 0; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_ValueError, "fits file is NULL"); + return NULL; + } + + if (!PyArg_ParseTuple(args, (char *)"iO", &hdunum, &rows_obj)) { + return NULL; + } + rows_array = (PyArrayObject *)rows_obj; + + rows = (LONGLONG *)PyArray_DATA(rows_array); + nrows = PyArray_SIZE(rows_array); + if (nrows <= 0) { + Py_RETURN_NONE; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_delete_rowlistll(self->fits, rows, nrows, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this does a full close and reopen + if (fits_flush_file(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +/* + * read a single, entire column from an ascii table into the input array. This + * version uses the standard read column instead of our by-bytes version. + * + * A number of assumptions are made, such as that columns are scalar, which + * is true for ascii. + */ + +static int read_ascii_column_all(fitsfile *fits, int colnum, + PyArrayObject *array, int *status) { + + int npy_dtype = 0; + int fits_dtype = 0; + + npy_intp nelem = 0; + LONGLONG firstelem = 1; + LONGLONG firstrow = 1; + int *anynul = NULL; + void *nulval = 0; + char *nulstr = " "; + void *data = NULL; + char *cdata = NULL; + + npy_dtype = PyArray_TYPE(array); + fits_dtype = npy_to_fits_table_type(npy_dtype, 0); + + nelem = PyArray_SIZE(array); + + if (fits_dtype == TSTRING) { + npy_intp i = 0; + LONGLONG rownum = 0; + + for (i = 0; i < nelem; i++) { + cdata = PyArray_GETPTR1(array, i); + rownum = (LONGLONG)(1 + i); + if (fits_read_col_str(fits, colnum, rownum, firstelem, 1, nulstr, + &cdata, anynul, status) > 0) { + return 1; + } + } + + /* + + LONGLONG twidth=0; + char** strdata=NULL; + + cdata = (char*) PyArray_DATA(array); + + strdata=malloc(nelem*sizeof(char*)); + if (NULL==strdata) { + PyErr_SetString(PyExc_MemoryError, "could not allocate temporary + string pointers"); *status = 99; return 1; + + } + + + twidth=fits->Fptr->tableptr[colnum-1].twidth; + for (i=0; i 0) { free(strdata); return 1; + } + + free(strdata); + */ + + } else { + data = PyArray_DATA(array); + if (fits_read_col(fits, fits_dtype, colnum, firstrow, firstelem, nelem, + nulval, data, anynul, status) > 0) { + return 1; + } + } + + return 0; +} +static int read_ascii_column_byrow(fitsfile *fits, int colnum, + PyArrayObject *array, PyArrayObject *rows, + PyArrayObject *sortind, int *status) { + + int npy_dtype = 0; + int fits_dtype = 0; + + npy_intp nelem = 0; + LONGLONG firstelem = 1; + LONGLONG rownum = 0; + npy_int64 si = 0; + npy_intp nrows = -1; + + int *anynul = NULL; + void *nulval = 0; + char *nulstr = " "; + void *data = NULL; + char *cdata = NULL; + + int dorows = 0; + + npy_intp i = 0; + + npy_dtype = PyArray_TYPE(array); + fits_dtype = npy_to_fits_table_type(npy_dtype, 0); + + nelem = PyArray_SIZE(array); + + if ((PyObject *)rows != Py_None) { + dorows = 1; + nrows = PyArray_SIZE(rows); + if (nrows != nelem) { + PyErr_Format(PyExc_ValueError, + "input array[%ld] and rows[%ld] have different size", + nelem, nrows); + return 1; + } + } + + for (i = 0; i < nrows; i++) { + if (dorows) { + si = *(npy_int64 *)PyArray_GETPTR1(sortind, i); + rownum = (LONGLONG)(*(npy_int64 *)PyArray_GETPTR1(rows, si)); + rownum += 1; + } else { + rownum = (LONGLONG)(1 + i); + } + // assuming 1-D + data = PyArray_GETPTR1(array, si); + if (fits_dtype == TSTRING) { + cdata = (char *)data; + if (fits_read_col_str(fits, colnum, rownum, firstelem, 1, nulstr, + &cdata, anynul, status) > 0) { + return 1; + } + } else { + if (fits_read_col(fits, fits_dtype, colnum, rownum, firstelem, 1, + nulval, data, anynul, status) > 0) { + return 1; + } + } + } + + return 0; +} + +static int read_ascii_column(fitsfile *fits, int colnum, PyArrayObject *array, + PyArrayObject *rows, PyArrayObject *sortind, + int *status) { + + int ret = 0; + if ((PyObject *)rows != Py_None || !PyArray_ISCONTIGUOUS(array)) { + ret = + read_ascii_column_byrow(fits, colnum, array, rows, sortind, status); + } else { + ret = read_ascii_column_all(fits, colnum, array, status); + } + + return ret; +} + +// read a subset of rows for the input column +// the row array is assumed to be unique and sorted. +static int read_binary_column(fitsfile *fits, int colnum, npy_intp nrows, + npy_int64 *rows, npy_int64 *sortind, void *vdata, + npy_intp stride, int *status) { + + FITSfile *hdu = NULL; + tcolumn *colptr = NULL; + LONGLONG file_pos = 0, irow = 0; + npy_int64 row = 0, si = 0; + + LONGLONG repeat = 0; + LONGLONG width = 0; + + // use char for pointer arith. It's actually ok to use void as char but + // this is just in case. + char *data = NULL, *ptr = NULL; + + data = (char *)vdata; + + // using struct defs here, could cause problems + hdu = fits->Fptr; + colptr = hdu->tableptr + (colnum - 1); + + repeat = colptr->trepeat; + width = colptr->tdatatype == TSTRING ? 1 : colptr->twidth; + + for (irow = 0; irow < nrows; irow++) { + if (rows != NULL) { + si = sortind[irow]; + row = rows[si]; + } else { + row = irow; + } + + ptr = data + si * stride; + + file_pos = hdu->datastart + row * hdu->rowlength + colptr->tbcol; + ffmbyt(fits, file_pos, REPORT_EOF, status); + if (ffgbytoff(fits, width, repeat, 0, (void *)ptr, status)) { + return 1; + } + } + + return 0; +} + +/* + * read from a column into an input array + */ +static PyObject *PyFITSObject_read_column(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + int colnum = 0; + + FITSfile *hdu = NULL; + int status = 0; + + PyObject *array_obj = NULL, *rows_obj = NULL, *sortind_obj = NULL; + PyArrayObject *array = NULL, *rows_array = NULL, *sortind_array = NULL; + + if (!PyArg_ParseTuple(args, (char *)"iiOOO", &hdunum, &colnum, &array_obj, + &rows_obj, &sortind_obj)) { + return NULL; + } + + array = (PyArrayObject *)array_obj; + rows_array = (PyArrayObject *)rows_obj; + sortind_array = (PyArrayObject *)sortind_obj; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // using struct defs here, could cause problems + hdu = self->fits->Fptr; + if (hdutype == IMAGE_HDU) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot yet read columns from an IMAGE_HDU"); + return NULL; + } + if (colnum < 1 || colnum > hdu->tfield) { + PyErr_SetString(PyExc_RuntimeError, + "requested column is out of bounds"); + return NULL; + } + + if (hdutype == ASCII_TBL) { + if (read_ascii_column(self->fits, colnum, array, rows_array, + sortind_array, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + } else { + void *data = PyArray_DATA(array); + npy_intp nrows = 0, nsortind = 0; + npy_int64 *rows = NULL, *sortind = NULL; + npy_intp stride = PyArray_STRIDE(array, 0); + if (rows_obj == Py_None) { + nrows = hdu->numrows; + } else { + rows = get_int64_from_array(rows_array, &nrows); + sortind = get_int64_from_array(sortind_array, &nsortind); + } + + if (read_binary_column(self->fits, colnum, nrows, rows, sortind, data, + stride, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + } + Py_RETURN_NONE; +} + +/* + * Free all the elements in the python list as well as the list itself + */ +static void free_all_python_list(PyObject *list) { + if (PyList_Check(list)) { + Py_ssize_t i = 0; + for (i = 0; i < PyList_Size(list); i++) { + Py_XDECREF(PyList_GetItem(list, i)); + } + } + Py_XDECREF(list); +} + +static PyObject *read_var_string(fitsfile *fits, int colnum, LONGLONG row, + LONGLONG nchar, int *status) { + LONGLONG firstelem = 1; + char *str = NULL; + char *strarr[1]; + PyObject *stringObj = NULL; + void *nulval = 0; + int *anynul = NULL; + + str = calloc(nchar + 1, sizeof(char)); + if (str == NULL) { + PyErr_Format(PyExc_MemoryError, + "Could not allocate string of size %lld", nchar); + return NULL; + } + + strarr[0] = str; + if (fits_read_col(fits, TSTRING, colnum, row, firstelem, nchar, nulval, + strarr, anynul, status) > 0) { + goto read_var_string_cleanup; + } +#if PY_MAJOR_VERSION >= 3 + // bytes + stringObj = Py_BuildValue("y", str); +#else + stringObj = Py_BuildValue("s", str); +#endif + if (NULL == stringObj) { + PyErr_Format(PyExc_MemoryError, + "Could not allocate py string of size %lld", nchar); + goto read_var_string_cleanup; + } + +read_var_string_cleanup: + free(str); + + return stringObj; +} +static PyObject *read_var_nums(fitsfile *fits, int colnum, LONGLONG row, + LONGLONG nelem, int fits_dtype, int npy_dtype, + int *status) { + LONGLONG firstelem = 1; + PyArrayObject *array = NULL; + void *nulval = 0; + int *anynul = NULL; + npy_intp dims[1]; + int fortran = 0; + void *data = NULL; + + dims[0] = nelem; + array = (PyArrayObject *)PyArray_ZEROS(1, dims, npy_dtype, fortran); + if (array == NULL) { + PyErr_Format(PyExc_MemoryError, + "Could not allocate array type %d size %lld", npy_dtype, + nelem); + return NULL; + } + data = PyArray_DATA(array); + if (fits_read_col(fits, abs(fits_dtype), colnum, row, firstelem, nelem, + nulval, data, anynul, status) > 0) { + Py_XDECREF(array); + return NULL; + } + + return (PyObject *)array; +} +/* + * read a variable length column as a list of arrays + * what about strings? + */ +static PyObject *PyFITSObject_read_var_column_as_list(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int colnum = 0; + PyObject *rows_obj = NULL, *sortind_obj = NULL; + PyArrayObject *rows_array = NULL, *sortind_array = NULL; + + int hdutype = 0; + int ncols = 0; + const npy_int64 *rows = NULL, *sortind = NULL; + LONGLONG nrows = 0; + int get_all_rows = 0; + + int status = 0, tstatus = 0; + + int fits_dtype = 0; + int npy_dtype = 0; + int isvariable = 0; + LONGLONG repeat = 0; + LONGLONG width = 0; + LONGLONG offset = 0; + LONGLONG i = 0; + LONGLONG row = 0; + npy_int64 si = 0; + + PyObject *listObj = NULL; + PyObject *tempObj = NULL; + + if (!PyArg_ParseTuple(args, (char *)"iiOO", &hdunum, &colnum, &rows_obj, + &sortind_obj)) { + return NULL; + } + rows_array = (PyArrayObject *)rows_obj; + sortind_array = (PyArrayObject *)sortind_obj; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (hdutype == IMAGE_HDU) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot yet read columns from an IMAGE_HDU"); + return NULL; + } + // using struct defs here, could cause problems + fits_get_num_cols(self->fits, &ncols, &status); + if (colnum < 1 || colnum > ncols) { + PyErr_SetString(PyExc_RuntimeError, + "requested column is out of bounds"); + return NULL; + } + + if (fits_get_coltypell(self->fits, colnum, &fits_dtype, &repeat, &width, + &status) > 0) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + npy_dtype = fits_to_npy_table_type(fits_dtype, &isvariable); + if (npy_dtype < 0) { + return NULL; + } + if (!isvariable) { + PyErr_Format(PyExc_TypeError, "Column %d not a variable length %d", + colnum, fits_dtype); + return NULL; + } + + if (rows_obj == Py_None) { + fits_get_num_rowsll(self->fits, &nrows, &tstatus); + get_all_rows = 1; + } else { + npy_intp tnrows = 0, nsortind = 0; + rows = (const npy_int64 *)get_int64_from_array(rows_array, &tnrows); + sortind = + (const npy_int64 *)get_int64_from_array(sortind_array, &nsortind); + nrows = (LONGLONG)tnrows; + get_all_rows = 0; + } + + listObj = PyList_New(0); + + for (i = 0; i < nrows; i++) { + tempObj = NULL; + + if (get_all_rows) { + row = i + 1; + } else { + si = sortind[i]; + row = (LONGLONG)(rows[si] + 1); + } + + // repeat holds how many elements are in this row + if (fits_read_descriptll(self->fits, colnum, row, &repeat, &offset, + &status) > 0) { + goto read_var_column_cleanup; + } + + if (fits_dtype == -TSTRING) { + tempObj = read_var_string(self->fits, colnum, row, repeat, &status); + } else { + tempObj = read_var_nums(self->fits, colnum, row, repeat, fits_dtype, + npy_dtype, &status); + } + if (tempObj == NULL) { + tstatus = 1; + goto read_var_column_cleanup; + } + PyList_Append(listObj, tempObj); + Py_XDECREF(tempObj); + } + +read_var_column_cleanup: + + if (status != 0 || tstatus != 0) { + Py_XDECREF(tempObj); + free_all_python_list(listObj); + if (status != 0) { + set_ioerr_string_from_status(status, self); + } + return NULL; + } + + return listObj; +} + +// read specified columns and rows +static int read_binary_rec_columns(fitsfile *fits, npy_intp ncols, + npy_int64 *colnums, npy_intp nrows, + npy_int64 *rows, npy_int64 *sortind, + PyArrayObject *array, int *status) { + FITSfile *hdu = NULL; + tcolumn *colptr = NULL; + LONGLONG file_pos = 0; + npy_intp col = 0; + npy_int64 colnum = 0; + char *ptr = NULL; + + npy_intp irow = 0; + npy_int64 row = 0, si = 0; + + LONGLONG gsize = 0; // number of bytes in column + LONGLONG repeat = 0; + LONGLONG width = 0; + + // using struct defs here, could cause problems + hdu = fits->Fptr; + + for (irow = 0; irow < nrows; irow++) { + if (rows != NULL) { + si = sortind[irow]; + row = rows[si]; + } else { + si = irow; + row = irow; + } + + ptr = (char *)PyArray_GETPTR1(array, si); + for (col = 0; col < ncols; col++) { + + colnum = colnums[col]; + colptr = hdu->tableptr + (colnum - 1); + + repeat = colptr->trepeat; + width = colptr->tdatatype == TSTRING ? 1 : colptr->twidth; + + file_pos = hdu->datastart + row * hdu->rowlength + colptr->tbcol; + + if (colptr->tdatatype == TBIT) { + if (fits_read_col_bit(fits, colnum, row + 1, 1, repeat, ptr, + status)) { + return 1; + } + } else { + // can just do one status check, since status are inherited. + ffmbyt(fits, file_pos, REPORT_EOF, status); + if (ffgbytoff(fits, width, repeat, 0, (void *)ptr, status)) { + return 1; + } + } + + gsize = repeat * width; + ptr += gsize; + } + } + + return 0; +} + +// python method for reading specified columns and rows +static PyObject *PyFITSObject_read_columns_as_rec(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + npy_intp ncols = 0; + npy_int64 *colnums = NULL; + FITSfile *hdu = NULL; + + int status = 0; + + PyObject *columns_obj = NULL, *array_obj = NULL, *rows_obj = NULL, + *sortind_obj = NULL; + + npy_intp nrows, nsortind; + npy_int64 *rows = NULL, *sortind = NULL; + + if (!PyArg_ParseTuple(args, (char *)"iOOOO", &hdunum, &columns_obj, + &array_obj, &rows_obj, &sortind_obj)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + goto recread_columns_cleanup; + } + + if (hdutype == IMAGE_HDU) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot read IMAGE_HDU into a recarray"); + return NULL; + } + + colnums = get_int64_from_array((PyArrayObject *)columns_obj, &ncols); + if (colnums == NULL) { + return NULL; + } + + hdu = self->fits->Fptr; + + if (rows_obj == Py_None) { + nrows = hdu->numrows; + } else { + rows = get_int64_from_array((PyArrayObject *)rows_obj, &nrows); + if (rows == NULL) { + return NULL; + } + sortind = get_int64_from_array((PyArrayObject *)sortind_obj, &nsortind); + if (sortind == NULL) { + return NULL; + } + } + if (read_binary_rec_columns(self->fits, ncols, colnums, nrows, rows, + sortind, (PyArrayObject *)array_obj, &status)) { + goto recread_columns_cleanup; + } + +recread_columns_cleanup: + + if (status != 0) { + set_ioerr_string_from_status(status, self); + return NULL; + } + Py_RETURN_NONE; +} + +/* + * read specified columns and rows + * + * Move by offset instead of just groupsize; this allows us to read into a + * recarray while skipping some fields, e.g. variable length array fields, to + * be read separately. + * + * If rows is NULL, then nrows are read consecutively. + */ + +static int read_columns_as_rec_byoffset( + fitsfile *fits, npy_intp ncols, + const npy_int64 *colnums, // columns to read from file + const npy_int64 + *field_offsets, // offsets of corresponding fields within array + npy_intp nrows, const npy_int64 *rows, const npy_int64 *sortind, char *data, + npy_intp recsize, int *status) { + + FITSfile *hdu = NULL; + tcolumn *colptr = NULL; + LONGLONG file_pos = 0; + npy_intp col = 0; + npy_int64 colnum = 0; + + char *ptr = NULL; + + int get_all_rows = 1; + npy_intp irow = 0; + npy_int64 row = 0, si = 0; + + long groupsize = 0; // number of bytes in column + long ngroups = 1; // number to read, one for row-by-row reading + long group_gap = 0; // gap between groups, zero since we aren't using it + + if (rows != NULL) { + get_all_rows = 0; + } + + // using struct defs here, could cause problems + hdu = fits->Fptr; + for (irow = 0; irow < nrows; irow++) { + if (get_all_rows) { + row = irow; + si = irow; + } else { + si = sortind[irow]; + row = rows[si]; + } + for (col = 0; col < ncols; col++) { + + // point to this field in the array, allows for skipping + ptr = data + si * recsize + field_offsets[col]; + + colnum = colnums[col]; + colptr = hdu->tableptr + (colnum - 1); + + groupsize = get_groupsize(colptr); + + file_pos = hdu->datastart + row * hdu->rowlength + colptr->tbcol; + + // can just do one status check, since status are inherited. + ffmbyt(fits, file_pos, REPORT_EOF, status); + if (ffgbytoff(fits, groupsize, ngroups, group_gap, (void *)ptr, + status)) { + return 1; + } + } + } + + return 0; +} + +/* python method for reading specified columns and rows, moving by offset in + * the array to allow some fields not read. + * + * columnsObj is the columns in the fits file to read. + * offsetsObj is the offsets of the corresponding fields into the array. + */ +static PyObject * +PyFITSObject_read_columns_as_rec_byoffset(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + npy_intp ncols = 0; + npy_intp noffsets = 0; + npy_intp nrows = 0, nsortind = 0; + const npy_int64 *colnums = NULL; + const npy_int64 *offsets = NULL; + const npy_int64 *rows = NULL, *sortind = NULL; + + PyObject *columns_obj = NULL; + PyObject *offsets_obj = NULL; + PyObject *rows_obj = NULL; + PyObject *sortind_obj = NULL; + + PyObject *array_obj = NULL; + PyArrayObject *array = NULL; + void *data = NULL; + npy_intp recsize = 0; + + if (!PyArg_ParseTuple(args, (char *)"iOOOOO", &hdunum, &columns_obj, + &offsets_obj, &array_obj, &rows_obj, &sortind_obj)) { + return NULL; + } + + array = (PyArrayObject *)array_obj; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + goto recread_columns_byoffset_cleanup; + } + + if (hdutype == IMAGE_HDU) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot read IMAGE_HDU into a recarray"); + return NULL; + } + + colnums = (const npy_int64 *)get_int64_from_array( + (PyArrayObject *)columns_obj, &ncols); + if (colnums == NULL) { + return NULL; + } + offsets = (const npy_int64 *)get_int64_from_array( + (PyArrayObject *)offsets_obj, &noffsets); + if (offsets == NULL) { + return NULL; + } + if (noffsets != ncols) { + PyErr_Format(PyExc_ValueError, + "%ld columns requested but got %ld offsets", ncols, + noffsets); + return NULL; + } + + if (rows_obj != Py_None) { + rows = (const npy_int64 *)get_int64_from_array( + (PyArrayObject *)rows_obj, &nrows); + sortind = (const npy_int64 *)get_int64_from_array( + (PyArrayObject *)sortind_obj, &nsortind); + } else { + nrows = PyArray_SIZE(array); + } + + data = PyArray_DATA(array); + recsize = PyArray_ITEMSIZE(array); + if (read_columns_as_rec_byoffset(self->fits, ncols, colnums, offsets, nrows, + rows, sortind, (char *)data, recsize, + &status) > 0) { + goto recread_columns_byoffset_cleanup; + } + +recread_columns_byoffset_cleanup: + + if (status != 0) { + set_ioerr_string_from_status(status, self); + return NULL; + } + Py_RETURN_NONE; +} + +// read specified rows, all columns +static int read_rec_bytes_byrow(fitsfile *fits, npy_intp nrows, npy_int64 *rows, + npy_int64 *sortind, void *vdata, int *status) { + + FITSfile *hdu = NULL; + + npy_intp irow = 0, si = 0; + LONGLONG firstrow = 1; + LONGLONG firstchar = 1; + + // use char for pointer arith. It's actually ok to use void as char but + // this is just in case. + unsigned char *ptr, *data; + + // using struct defs here, could cause problems + hdu = fits->Fptr; + // ptr = (unsigned char*) data; + data = (unsigned char *)vdata; + + for (irow = 0; irow < nrows; irow++) { + + si = sortind[irow]; + + // Input is zero-offset + firstrow = 1 + (LONGLONG)rows[si]; + + ptr = data + si * hdu->rowlength; + + if (fits_read_tblbytes(fits, firstrow, firstchar, hdu->rowlength, ptr, + status)) { + return 1; + } + + // ptr += hdu->rowlength; + } + + return 0; +} +// read specified rows, all columns +/* +static int read_rec_bytes_byrowold( + fitsfile* fits, + npy_intp nrows, npy_int64* rows, + void* data, int* status) { + FITSfile* hdu=NULL; + LONGLONG file_pos=0; + + npy_intp irow=0; + npy_int64 row=0; + + // use char for pointer arith. It's actually ok to use void as char but + // this is just in case. + char* ptr; + + long ngroups=1; // number to read, one for row-by-row reading + long offset=0; // gap between groups, not stride. zero since we aren't +using it + + // using struct defs here, could cause problems + hdu = fits->Fptr; + ptr = (char*) data; + + for (irow=0; irowdatastart + row*hdu->rowlength; + + // can just do one status check, since status are inherited. + ffmbyt(fits, file_pos, REPORT_EOF, status); + if (ffgbytoff(fits, hdu->rowlength, ngroups, offset, (void*) ptr, +status)) { return 1; + } + ptr += hdu->rowlength; + } + + return 0; +} +*/ + +// python method to read all columns but subset of rows +static PyObject *PyFITSObject_read_rows_as_rec(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + + int status = 0; + void *data = NULL; + + PyObject *array_obj = NULL, *rows_obj = NULL, *sortind_obj = NULL; + npy_intp nrows = 0, nsortind = 0; + npy_int64 *rows = NULL; + npy_int64 *sortind = NULL; + + if (!PyArg_ParseTuple(args, (char *)"iOOO", &hdunum, &array_obj, &rows_obj, + &sortind_obj)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + goto recread_byrow_cleanup; + } + + if (hdutype == IMAGE_HDU) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot read IMAGE_HDU into a recarray"); + return NULL; + } + + data = PyArray_DATA((PyArrayObject *)array_obj); + + rows = get_int64_from_array((PyArrayObject *)rows_obj, &nrows); + if (rows == NULL) { + return NULL; + } + sortind = get_int64_from_array((PyArrayObject *)sortind_obj, &nsortind); + if (sortind == NULL) { + return NULL; + } + + if (read_rec_bytes_byrow(self->fits, nrows, rows, sortind, data, &status)) { + goto recread_byrow_cleanup; + } + +recread_byrow_cleanup: + + if (status != 0) { + set_ioerr_string_from_status(status, self); + return NULL; + } + Py_RETURN_NONE; +} + +/* Read the range of rows, 1-offset. It is assumed the data match the table + * perfectly. + */ + +static int read_rec_range(fitsfile *fits, LONGLONG firstrow, LONGLONG nrows, + void *data, int *status) { + // can also use this for reading row ranges + LONGLONG firstchar = 1; + LONGLONG nchars = 0; + + nchars = (fits->Fptr)->rowlength * nrows; + + if (fits_read_tblbytes(fits, firstrow, firstchar, nchars, + (unsigned char *)data, status)) { + return 1; + } + + return 0; +} + +/* here rows are 1-offset, unlike when reading a specific subset of rows */ +static PyObject *PyFITSObject_read_as_rec(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + + int status = 0; + PyObject *array_obj = NULL; + void *data = NULL; + + PY_LONG_LONG firstrow = 0; + PY_LONG_LONG lastrow = 0; + PY_LONG_LONG nrows = 0; + + if (!PyArg_ParseTuple(args, (char *)"iLLO", &hdunum, &firstrow, &lastrow, + &array_obj)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + goto recread_asrec_cleanup; + } + + if (hdutype == IMAGE_HDU) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot read IMAGE_HDU into a recarray"); + return NULL; + } + + data = PyArray_DATA((PyArrayObject *)array_obj); + + nrows = lastrow - firstrow + 1; + if (read_rec_range(self->fits, (LONGLONG)firstrow, (LONGLONG)nrows, data, + &status)) { + goto recread_asrec_cleanup; + } + +recread_asrec_cleanup: + + if (status != 0) { + set_ioerr_string_from_status(status, self); + return NULL; + } + Py_RETURN_NONE; +} + +static int fits_is_compressed_with_nulls(fitsfile *fits) { + int cstatus = 0, clstatus = 0, hstatus = 0; + int colnum, zblank; + if (fits_is_compressed_image(fits, &cstatus) && + (ffgcno(fits, CASEINSEN, "ZBLANK", &colnum, &clstatus) <= 0 || + ffgky(fits, TINT, "ZBLANK", &zblank, NULL, &hstatus) <= 0)) { + return 1; + } else { + return 0; + } +} + +// read an n-dimensional "image" into the input array. Only minimal checking +// of the input array is done. +// Note numpy allows a maximum of 32 dimensions +static PyObject *PyFITSObject_read_image(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + int status = 0; + PyObject *array_obj = NULL; + PyArrayObject *array = NULL; + void *data = NULL; + int npy_dtype = 0; + int dummy = 0, fits_read_dtype = 0; + + int maxdim = NUMPY_MAX_DIMS; // numpy maximum + int datatype = 0; // type info for axis + int naxis = 0; // number of axes + int i = 0; + LONGLONG naxes[NUMPY_MAX_DIMS]; + ; // size of each axis + LONGLONG firstpixels[NUMPY_MAX_DIMS]; + LONGLONG size = 0; + npy_intp arrsize = 0; + + int anynul = 0; + + if (!PyArg_ParseTuple(args, (char *)"iO", &hdunum, &array_obj)) { + return NULL; + } + + array = (PyArrayObject *)array_obj; + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + return NULL; + } + + if (fits_get_img_paramll(self->fits, maxdim, &datatype, &naxis, naxes, + &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + // make sure dims match + size = 0; + size = naxes[0]; + for (i = 1; i < naxis; i++) { + size *= naxes[i]; + } + arrsize = PyArray_SIZE(array); + data = PyArray_DATA(array); + + if (size != arrsize) { + PyErr_Format(PyExc_RuntimeError, + "Input array size is %ld but on disk array size is %lld", + arrsize, size); + return NULL; + } + + npy_dtype = PyArray_TYPE(array); + npy_to_fits_image_types(npy_dtype, &dummy, &fits_read_dtype); + + float fnullval = NAN; + double dnullval = NAN; + void *nullval_ptr = NULL; + + // we only set null checking for compressed images of + // floating point data + // nans works fine for non-compressed images and we do + // not consider int data + if (fits_is_compressed_with_nulls(self->fits)) { + if (fits_read_dtype == TFLOAT) { + nullval_ptr = (void *)(&fnullval); + } else if (fits_read_dtype == TDOUBLE) { + nullval_ptr = (void *)(&dnullval); + } + } + + for (i = 0; i < naxis; i++) { + firstpixels[i] = 1; + } + if (fits_read_pixll(self->fits, fits_read_dtype, firstpixels, size, + nullval_ptr, data, &anynul, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject *PyFITSObject_read_raw(struct PyFITSObject *self, + PyObject *args) { + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + // fitsfile* fits = self->fits; + FITSfile *FITS = self->fits->Fptr; + int status = 0; + char *filedata; + LONGLONG sz; + LONGLONG io_pos; + PyObject *stringobj; + + // Flush (close & reopen HDU) to make everything consistent + ffflus(self->fits, &status); + if (status) { + PyErr_Format(PyExc_RuntimeError, + "Failed to flush FITS file data to disk; CFITSIO code %i", + status); + return NULL; + } + // Allocate buffer for string + sz = FITS->filesize; + // Create python string object of requested size, unitialized + stringobj = PyBytes_FromStringAndSize(NULL, sz); + if (!stringobj) { + PyErr_Format( + PyExc_RuntimeError, + "Failed to allocate python string object to hold FITS file " + "data: %i bytes", + (int)sz); + return NULL; + } + // Grab pointer to the memory buffer of the python string object + filedata = PyBytes_AsString(stringobj); + if (!filedata) { + Py_DECREF(stringobj); + return NULL; + } + // Remember old file position + io_pos = FITS->io_pos; + // Seek to beginning of file + if (ffseek(FITS, 0)) { + Py_DECREF(stringobj); + PyErr_Format(PyExc_RuntimeError, + "Failed to seek to beginning of FITS file"); + return NULL; + } + // Read into filedata + if (ffread(FITS, sz, filedata, &status)) { + Py_DECREF(stringobj); + PyErr_Format(PyExc_RuntimeError, + "Failed to read file data into memory: CFITSIO code %i", + status); + return NULL; + } + // Seek back to where we were + if (ffseek(FITS, io_pos)) { + Py_DECREF(stringobj); + PyErr_Format(PyExc_RuntimeError, + "Failed to seek back to original FITS file position"); + return NULL; + } + return stringobj; +} + +static int get_long_slices(PyArrayObject *fpix_arr, PyArrayObject *lpix_arr, + PyArrayObject *step_arr, long **fpix, long **lpix, + long **step) { + + int i = 0; + npy_int64 *ptr = NULL; + npy_intp fsize = 0, lsize = 0, ssize = 0; + + fsize = PyArray_SIZE(fpix_arr); + lsize = PyArray_SIZE(lpix_arr); + ssize = PyArray_SIZE(step_arr); + + if (lsize != fsize || ssize != fsize) { + PyErr_SetString(PyExc_RuntimeError, "start/end/step must be same len"); + return 1; + } + + *fpix = calloc(fsize, sizeof(long)); + *lpix = calloc(fsize, sizeof(long)); + *step = calloc(fsize, sizeof(long)); + + for (i = 0; i < fsize; i++) { + ptr = PyArray_GETPTR1(fpix_arr, i); + (*fpix)[i] = *ptr; + ptr = PyArray_GETPTR1(lpix_arr, i); + (*lpix)[i] = *ptr; + ptr = PyArray_GETPTR1(step_arr, i); + (*step)[i] = *ptr; + } + return 0; +} + +// read an n-dimensional "image" into the input array. Only minimal checking +// of the input array is done. +static PyObject *PyFITSObject_read_image_slice(struct PyFITSObject *self, + PyObject *args) { + int hdunum = 0; + int hdutype = 0; + int status = 0; + PyObject *fpix_obj = NULL; + PyObject *lpix_obj = NULL; + PyObject *step_obj = NULL; + int ignore_scaling = FALSE; + PyObject *array = NULL; + long *fpix = NULL; + long *lpix = NULL; + long *step = NULL; + void *data = NULL; + int npy_dtype = 0; + int dummy = 0, fits_read_dtype = 0; + + int anynul = 0; + + if (!PyArg_ParseTuple(args, (char *)"iOOOiO", &hdunum, &fpix_obj, &lpix_obj, + &step_obj, &ignore_scaling, &array)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + return NULL; + } + + if (ignore_scaling == TRUE && + fits_set_bscale(self->fits, 1.0, 0.0, &status)) { + return NULL; + } + + if (get_long_slices((PyArrayObject *)fpix_obj, (PyArrayObject *)lpix_obj, + (PyArrayObject *)step_obj, &fpix, &lpix, &step)) { + return NULL; + } + data = PyArray_DATA((PyArrayObject *)array); + + npy_dtype = PyArray_TYPE((PyArrayObject *)array); + npy_to_fits_image_types(npy_dtype, &dummy, &fits_read_dtype); + + float fnullval = NAN; + double dnullval = NAN; + void *nullval_ptr = NULL; + + // we only set null checking for compressed images of + // floating point data + // nans works fine for non-compressed images and we do + // not consider int data + if (fits_is_compressed_with_nulls(self->fits)) { + if (fits_read_dtype == TFLOAT) { + nullval_ptr = (void *)(&fnullval); + } else if (fits_read_dtype == TDOUBLE) { + nullval_ptr = (void *)(&dnullval); + } + } + + if (fits_read_subset(self->fits, fits_read_dtype, fpix, lpix, step, + nullval_ptr, data, &anynul, &status)) { + set_ioerr_string_from_status(status, self); + goto read_image_slice_cleanup; + } + +read_image_slice_cleanup: + free(fpix); + free(lpix); + free(step); + + if (status != 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static int hierarch_is_string(const char *card) { + int i = 0, is_string_value = 1; + + for (i = 0; i < 78; i++) { + if (card[i] == '=') { + // we found the equals, now if it is a string we + // now exactly where the quote must be + if (card[i + 2] == '\'') { + is_string_value = 1; + } else { + is_string_value = 0; + } + } + } + return is_string_value; +} + +// read the entire header as list of dicts with name,value,comment and full +// card +static PyObject *PyFITSObject_read_header(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + int lcont = 0, lcomm = 0, ls = 0; + int tocomp = 0; + int is_comment_or_history = 0, is_blank_key = 0; + char *longstr = NULL; + + char keyname[FLEN_KEYWORD]; + char value[FLEN_VALUE]; + char comment[FLEN_COMMENT]; + char scomment[FLEN_COMMENT]; + char card[FLEN_CARD]; + long is_string_value = 0; + + LONGLONG lval = 0; + double dval = 0; + + int nkeys = 0, morekeys = 0, i = 0; + int has_equals = 0, has_quote = 0, was_converted = 0, is_hierarch = 0; + + PyObject *list = NULL; + PyObject *dict = NULL; // to hold the dict for each record + + lcont = strlen("CONTINUE"); + lcomm = strlen("COMMENT"); + + if (!PyArg_ParseTuple(args, (char *)"i", &hdunum)) { + return NULL; + } + + if (self->fits == NULL) { + PyErr_SetString(PyExc_RuntimeError, "FITS file is NULL"); + return NULL; + } + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_get_hdrspace(self->fits, &nkeys, &morekeys, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + list = PyList_New(0); + for (i = 0; i < nkeys; i++) { + + // the full card + if (fits_read_record(self->fits, i + 1, card, &status)) { + Py_XDECREF(list); + set_ioerr_string_from_status(status, self); + return NULL; + } + + // this just returns the character string stored in the header; we + // can eval in python + if (fits_read_keyn(self->fits, i + 1, keyname, value, scomment, + &status)) { + Py_XDECREF(list); + set_ioerr_string_from_status(status, self); + return NULL; + } + + ls = strlen(keyname); + tocomp = (ls < lcont) ? ls : lcont; + + is_blank_key = 0; + is_hierarch = 0; + if (ls == 0) { + is_blank_key = 1; + } else { + + // skip CONTINUE, we already read the data + if (strncmp(keyname, "CONTINUE", tocomp) == 0) { + continue; + } + + if (strncmp(keyname, "COMMENT", tocomp) == 0 || + strncmp(keyname, "HISTORY", tocomp) == 0) { + is_comment_or_history = 1; + + } else { + is_comment_or_history = 0; + + if (fits_read_key_longstr(self->fits, keyname, &longstr, + comment, &status)) { + Py_XDECREF(list); + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (strncmp(card, "HIERARCH", 8) == 0) { + is_hierarch = 1; + if (hierarch_is_string(card)) { + is_string_value = 1; + } else { + is_string_value = 0; + } + } else { + int j; + has_equals = (card[8] == '=') ? 1 : 0; + has_quote = 0; + // Look for 0 or more space characters followed by a quote + // character. + for (j = 10; j < FLEN_CARD; j++) { + if (card[j] == '\'') { + has_quote = 1; + break; + } else if (card[j] != ' ') { + break; + } + } + if (has_equals && has_quote) { + is_string_value = 1; + } else { + is_string_value = 0; + } + } + } + } + + dict = PyDict_New(); + convert_to_ascii(card); + add_string_to_dict(dict, "card_string", card); + + if (is_blank_key) { + add_none_to_dict(dict, "name"); + add_string_to_dict(dict, "value", ""); + convert_to_ascii(scomment); + add_string_to_dict(dict, "comment", scomment); + + } else if (is_comment_or_history) { + // comment or history + convert_to_ascii(scomment); + add_string_to_dict(dict, "name", keyname); + add_string_to_dict(dict, "value", scomment); + add_string_to_dict(dict, "comment", scomment); + + } else { + + if (is_hierarch) { + // if a key is hierarch, then any ascii character is allowed + // except + // *, ? and #. Thus we convert any of those (and any chars we + // find that don't correspond to something written, ascii <= 32 + // or 127) to an underscore if the key is converted, then we + // cannot parse it further with cfitsio + was_converted = + convert_keyword_to_allowed_ascii_template_and_nonascii_only( + keyname); + } else { + // for non-hierach keys, we cannot use the cfitsio functions if + // any character besides those in the fits conventions + // (A-Z,a-z,0-9,_,-) are present. Thus we flag those and store + // their values as a string if this happens. + was_converted = has_invalid_keyword_chars(keyname); + + // in order to actually store the key in the python dict, we + // have to cut out any non-ascii chars we additionally convert + // the template chars to that the fits data we make can be + // written back without error note that the check by + // has_invalid_keyword_chars is more stringent than the checks + // done here, so if any conversion is done it has already been + // flagged above. + convert_keyword_to_allowed_ascii_template_and_nonascii_only( + keyname); + } + add_string_to_dict(dict, "name", keyname); + convert_to_ascii(comment); + add_string_to_dict(dict, "comment", comment); + + // if not a comment but empty value, put in None + tocomp = (ls < lcomm) ? ls : lcomm; + // if (!is_string_value && 0==strlen(longstr) && !is_comment) { + if (!is_string_value && 0 == strlen(longstr)) { + + add_none_to_dict(dict, "value"); + + } else { + + // if it's a string we just store it. + if (is_string_value) { + convert_to_ascii(longstr); + add_string_to_dict(dict, "value", longstr); + } else if (longstr[0] == 'T') { + add_true_to_dict(dict, "value"); + } else if (longstr[0] == 'F') { + add_false_to_dict(dict, "value"); + } else if (was_converted) { + // if we had to convert bad characters in the keyword name + // we can't attempt to get a numerical value using + // fits_read_key because some characters in a keyword name + // cause a seg fault + convert_to_ascii(longstr); + add_string_to_dict(dict, "value", longstr); + } else if ((strchr(longstr, '.') != NULL) || + (strchr(longstr, 'E') != NULL) || + (strchr(longstr, 'e') != NULL)) { + // we found a floating point value + fits_read_key(self->fits, TDOUBLE, keyname, &dval, comment, + &status); + add_double_to_dict(dict, "value", dval); + } else { + + // we might have found an integer + if (fits_read_key(self->fits, TLONGLONG, keyname, &lval, + comment, &status)) { + + // something non standard, just store it as a string + convert_to_ascii(longstr); + add_string_to_dict(dict, "value", longstr); + status = 0; + + } else { + add_long_long_to_dict(dict, "value", (long long)lval); + } + } + } + } + + free(longstr); + longstr = NULL; + + PyList_Append(list, dict); + Py_XDECREF(dict); + } + + return list; +} + +static PyObject *PyFITSObject_write_checksum(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + unsigned long datasum = 0; + unsigned long hdusum = 0; + + PyObject *dict = NULL; + + if (!PyArg_ParseTuple(args, (char *)"i", &hdunum)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_write_chksum(self->fits, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + if (fits_get_chksum(self->fits, &datasum, &hdusum, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + dict = PyDict_New(); + add_long_long_to_dict(dict, "datasum", (long long)datasum); + add_long_long_to_dict(dict, "hdusum", (long long)hdusum); + + return dict; +} +static PyObject *PyFITSObject_verify_checksum(struct PyFITSObject *self, + PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + + int dataok = 0, hduok = 0; + + PyObject *dict = NULL; + + if (!PyArg_ParseTuple(args, (char *)"i", &hdunum)) { + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + if (fits_verify_chksum(self->fits, &dataok, &hduok, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + dict = PyDict_New(); + add_long_to_dict(dict, "dataok", (long)dataok); + add_long_to_dict(dict, "hduok", (long)hduok); + + return dict; +} + +static PyObject *PyFITSObject_where(struct PyFITSObject *self, PyObject *args) { + int status = 0; + int hdunum = 0; + int hdutype = 0; + char *expression = NULL; + + long firstrow; + long nrows; + long ngood = 0; + char *row_status = NULL; + + // Indices of rows for which expression is true + PyObject *indices_obj = NULL; + int ndim = 1; + npy_intp dims[1]; + npy_intp *data = NULL; + long i = 0; + + if (!PyArg_ParseTuple(args, (char *)"isll", &hdunum, &expression, &firstrow, + &nrows)) { + return NULL; + } + + if (firstrow < 1 || nrows < 1) { + PyErr_SetString(PyExc_ValueError, + "firstrow and nrows must be positive integers"); + return NULL; + } + + if (fits_movabs_hdu(self->fits, hdunum, &hdutype, &status)) { + set_ioerr_string_from_status(status, self); + return NULL; + } + + row_status = malloc(nrows * sizeof(char)); + if (row_status == NULL) { + PyErr_SetString(PyExc_MemoryError, + "Could not allocate row_status array"); + return NULL; + } + + if (fits_find_rows(self->fits, expression, firstrow, nrows, &ngood, + row_status, &status)) { + set_ioerr_string_from_status(status, self); + goto where_function_cleanup; + } + + dims[0] = ngood; + indices_obj = PyArray_EMPTY(ndim, dims, NPY_INTP, 0); + if (indices_obj == NULL) { + PyErr_SetString(PyExc_MemoryError, "Could not allocate index array"); + goto where_function_cleanup; + } + + if (ngood > 0) { + data = PyArray_DATA((PyArrayObject *)indices_obj); + + for (i = 0; i < nrows; i++) { + if (row_status[i]) { + *data = (npy_intp)i; + data++; + } + } + } +where_function_cleanup: + free(row_status); + return indices_obj; +} + +// generic functions, not tied to an object + +static PyObject *PyFITS_cfitsio_version(void) { + float version = 0; + fits_get_version(&version); + return PyFloat_FromDouble((double)version); +} + +static PyObject *PyFITS_cfitsio_is_bundled(void) { +#ifdef FITSIO_USING_SYSTEM_FITSIO + Py_RETURN_FALSE; +#else + Py_RETURN_TRUE; +#endif +} + +static PyObject *PyFITS_cfitsio_has_bzip2_support(void) { +#ifdef FITSIO_HAS_BZIP2_SUPPORT + Py_RETURN_TRUE; +#else + Py_RETURN_FALSE; +#endif +} + +static PyObject *PyFITS_cfitsio_has_curl_support(void) { +#ifdef FITSIO_HAS_CURL_SUPPORT + Py_RETURN_TRUE; +#else + Py_RETURN_FALSE; +#endif +} + +static PyObject *PyFITS_cfitsio_null_value_for_nan(void) { + return PyFloat_FromDouble((double)INFINITY); +} + +/* + +'C', 'L', 'I', 'F' 'X' +character string, logical, integer, floating point, complex + +*/ + +static PyObject *PyFITS_get_keytype(PyObject *self, PyObject *args) { + + int status = 0; + char *card = NULL; + char dtype[2] = {0}; + + if (!PyArg_ParseTuple(args, (char *)"s", &card)) { + return NULL; + } + + if (fits_get_keytype(card, dtype, &status)) { + set_ioerr_string_from_status(status, NULL); + return NULL; + } else { + return Py_BuildValue("s", dtype); + } +} +static PyObject *PyFITS_get_key_meta(PyObject *self, PyObject *args) { + + int status = 0; + char *card = NULL; + char dtype[2] = {0}; + int keyclass = 0; + + if (!PyArg_ParseTuple(args, (char *)"s", &card)) { + return NULL; + } + + keyclass = fits_get_keyclass(card); + + if (fits_get_keytype(card, dtype, &status)) { + set_ioerr_string_from_status(status, NULL); + return NULL; + } + + return Py_BuildValue("is", keyclass, dtype); +} + +/* + + note the special first four comment fields will not be called comment but + structural! That will cause an exception to be raised, so the card should + be checked before calling this function + +*/ + +static PyObject *PyFITS_parse_card(PyObject *self, PyObject *args) { + + int status = 0; + char name[FLEN_VALUE] = {0}; + char value[FLEN_VALUE] = {0}; + char comment[FLEN_COMMENT] = {0}; + int keylen = 0; + int keyclass = 0; + int is_undefined = 0; + + char *card = NULL; + char dtype[2] = {0}; + PyObject *output = NULL; + + if (!PyArg_ParseTuple(args, (char *)"s", &card)) { + goto bail; + } + + keyclass = fits_get_keyclass(card); + + // only proceed if not comment or history, but note the special first four + // comment fields will not be called comment but structural! That will + // cause an exception to be raised, so the card should be checked before + // calling this function + + if (keyclass != TYP_COMM_KEY && keyclass != TYP_CONT_KEY) { + + if (fits_get_keyname(card, name, &keylen, &status)) { + set_ioerr_string_from_status(status, NULL); + goto bail; + } + if (fits_parse_value(card, value, comment, &status)) { + set_ioerr_string_from_status(status, NULL); + goto bail; + } + if (fits_get_keytype(value, dtype, &status)) { + + if (status == VALUE_UNDEFINED) { + is_undefined = 1; + status = 0; + } else { + set_ioerr_string_from_status(status, NULL); + goto bail; + } + } + } + +bail: + if (status != 0) { + return NULL; + } + + if (is_undefined) { + output = Py_BuildValue("isss", keyclass, name, dtype, comment); + } else { + output = Py_BuildValue("issss", keyclass, name, value, dtype, comment); + } + return output; +} + +static PyMethodDef PyFITSObject_methods[] = { + {"filename", (PyCFunction)PyFITSObject_filename, METH_VARARGS, + "filename\n\nReturn the name of the file."}, + + {"where", (PyCFunction)PyFITSObject_where, METH_VARARGS, + "where\n\nReturn an index array where the input expression evaluates to " + "true."}, + + {"movabs_hdu", (PyCFunction)PyFITSObject_movabs_hdu, METH_VARARGS, + "movabs_hdu\n\nMove to the specified HDU."}, + {"movnam_hdu", (PyCFunction)PyFITSObject_movnam_hdu, METH_VARARGS, + "movnam_hdu\n\nMove to the specified HDU by name and return the hdu " + "number."}, + + {"get_hdu_name_version", (PyCFunction)PyFITSObject_get_hdu_name_version, + METH_VARARGS, "get_hdu_name_version\n\nReturn a tuple (extname,extvers)."}, + {"get_hdu_info", (PyCFunction)PyFITSObject_get_hdu_info, METH_VARARGS, + "get_hdu_info\n\nReturn a dict with info about the specified HDU."}, + {"read_raw", (PyCFunction)PyFITSObject_read_raw, METH_NOARGS, + "read_raw\n\nRead the entire raw contents of the FITS file, returning a " + "python string."}, + {"read_image", (PyCFunction)PyFITSObject_read_image, METH_VARARGS, + "read_image\n\nRead the entire n-dimensional image array. No checking of " + "array is done."}, + {"read_image_slice", (PyCFunction)PyFITSObject_read_image_slice, + METH_VARARGS, "read_image_slice\n\nRead an image slice."}, + {"read_column", (PyCFunction)PyFITSObject_read_column, METH_VARARGS, + "read_column\n\nRead the column into the input array. No checking of " + "array is done."}, + {"read_var_column_as_list", + (PyCFunction)PyFITSObject_read_var_column_as_list, METH_VARARGS, + "read_var_column_as_list\n\nRead the variable length column as a list of " + "arrays."}, + {"read_columns_as_rec", (PyCFunction)PyFITSObject_read_columns_as_rec, + METH_VARARGS, + "read_columns_as_rec\n\nRead the specified columns into the input rec " + "array. No checking of array is done."}, + {"read_columns_as_rec_byoffset", + (PyCFunction)PyFITSObject_read_columns_as_rec_byoffset, METH_VARARGS, + "read_columns_as_rec_byoffset\n\nRead the specified columns into the " + "input rec array at the specified offsets. No checking of array is " + "done."}, + {"read_rows_as_rec", (PyCFunction)PyFITSObject_read_rows_as_rec, + METH_VARARGS, + "read_rows_as_rec\n\nRead the subset of rows into the input rec array. " + "No checking of array is done."}, + {"read_as_rec", (PyCFunction)PyFITSObject_read_as_rec, METH_VARARGS, + "read_as_rec\n\nRead a set of rows into the input rec array. No " + "significant checking of array is done."}, + {"read_header", (PyCFunction)PyFITSObject_read_header, + METH_VARARGS | METH_VARARGS, + "read_header\n\nRead the entire header as a list of dictionaries."}, + + {"create_image_hdu", (PyCFunction)PyFITSObject_create_image_hdu, + METH_VARARGS | METH_KEYWORDS, + "create_image_hdu\n\nWrite the input image to a new extension."}, + {"create_table_hdu", (PyCFunction)PyFITSObject_create_table_hdu, + METH_VARARGS | METH_KEYWORDS, + "create_table_hdu\n\nCreate a new table with the input parameters."}, + {"insert_col", (PyCFunction)PyFITSObject_insert_col, + METH_VARARGS | METH_KEYWORDS, "insert_col\n\nInsert a new column."}, + + {"write_checksum", (PyCFunction)PyFITSObject_write_checksum, METH_VARARGS, + "write_checksum\n\nCompute and write the checksums into the header."}, + {"verify_checksum", (PyCFunction)PyFITSObject_verify_checksum, METH_VARARGS, + "verify_checksum\n\nReturn a dict with dataok and hduok."}, + + {"reshape_image", (PyCFunction)PyFITSObject_reshape_image, METH_VARARGS, + "reshape_image\n\nReshape the image."}, + {"write_image", (PyCFunction)PyFITSObject_write_image, METH_VARARGS, + "write_image\n\nWrite the input image to a new extension."}, + {"write_subset", (PyCFunction)PyFITSObject_write_subset, METH_VARARGS, + "write_subset\n\nWrite a rectangular subset to the image."}, + //{"write_column", (PyCFunction)PyFITSObject_write_column, + // METH_VARARGS | METH_KEYWORDS, "write_column\n\nWrite a column into the + // specified hdu."}, + {"write_columns", (PyCFunction)PyFITSObject_write_columns, + METH_VARARGS | METH_KEYWORDS, + "write_columns\n\nWrite columns into the specified hdu."}, + {"write_var_column", (PyCFunction)PyFITSObject_write_var_column, + METH_VARARGS | METH_KEYWORDS, + "write_var_column\n\nWrite a variable length column into the specified " + "hdu from an object array."}, + {"write_record", (PyCFunction)PyFITSObject_write_record, METH_VARARGS, + "write_record\n\nWrite a header card."}, + {"write_string_key", (PyCFunction)PyFITSObject_write_string_key, + METH_VARARGS, + "write_string_key\n\nWrite a string key into the specified HDU."}, + {"write_double_key", (PyCFunction)PyFITSObject_write_double_key, + METH_VARARGS, + "write_double_key\n\nWrite a double key into the specified HDU."}, + + {"write_long_long_key", (PyCFunction)PyFITSObject_write_long_long_key, + METH_VARARGS, + "write_long_long_key\n\nWrite a long long key into the specified HDU."}, + {"write_logical_key", (PyCFunction)PyFITSObject_write_logical_key, + METH_VARARGS, + "write_logical_key\n\nWrite a logical key into the specified HDU."}, + + {"write_comment", (PyCFunction)PyFITSObject_write_comment, METH_VARARGS, + "write_comment\n\nWrite a comment into the header of the specified HDU."}, + {"write_history", (PyCFunction)PyFITSObject_write_history, METH_VARARGS, + "write_history\n\nWrite history into the header of the specified HDU."}, + {"write_continue", (PyCFunction)PyFITSObject_write_continue, METH_VARARGS, + "write_continue\n\nWrite contineu into the header of the specified HDU."}, + + {"write_undefined_key", (PyCFunction)PyFITSObject_write_undefined_key, + METH_VARARGS, + "write_undefined_key\n\nWrite a key without a value field into the header " + "of the specified HDU."}, + + {"delete_key", (PyCFunction)PyFITSObject_delete_key, METH_VARARGS, + "delete_key\n\nDeleta a key from the header of the specified HDU."}, + + {"insert_rows", (PyCFunction)PyFITSObject_insert_rows, METH_VARARGS, + "Insert blank rows"}, + + {"delete_row_range", (PyCFunction)PyFITSObject_delete_row_range, + METH_VARARGS, "Delete a range of rows"}, + {"delete_rows", (PyCFunction)PyFITSObject_delete_rows, METH_VARARGS, + "Delete a set of rows"}, + + {"close", (PyCFunction)PyFITSObject_close, METH_VARARGS, + "close\n\nClose the fits file."}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject PyFITSType = { +#if PY_MAJOR_VERSION >= 3 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ +#endif + "_fitsio.FITS", /*tp_name*/ + sizeof(struct PyFITSObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)PyFITSObject_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + // 0, /*tp_repr*/ + (reprfunc)PyFITSObject_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "FITSIO Class", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyFITSObject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + // 0, /* tp_init */ + (initproc)PyFITSObject_init, /* tp_init */ + 0, /* tp_alloc */ + // PyFITSObject_new, /* tp_new */ + PyType_GenericNew, /* tp_new */ +}; + +static PyMethodDef fitstype_methods[] = { + {"cfitsio_version", (PyCFunction)PyFITS_cfitsio_version, METH_NOARGS, + "cfitsio_version\n\nReturn the cfitsio version."}, + {"cfitsio_is_bundled", (PyCFunction)PyFITS_cfitsio_is_bundled, METH_NOARGS, + "cfitsio_is_bundled\n\nReturn True if library was built with a bundled " + "copy of cfitsio."}, + {"cfitsio_has_bzip2_support", (PyCFunction)PyFITS_cfitsio_has_bzip2_support, + METH_NOARGS, + "cfitsio_has_bzip2_support\n\nReturn True if cfitsio has support for " + "bzip2."}, + {"cfitsio_has_curl_support", (PyCFunction)PyFITS_cfitsio_has_curl_support, + METH_NOARGS, + "cfitsio_has_curl_support\n\nReturn True if cfitsio has support for " + "curl."}, + {"cfitsio_null_value_for_nan", + (PyCFunction)PyFITS_cfitsio_null_value_for_nan, METH_NOARGS, + "cfitsio_null_value_for_nan\n\nReturn our default null value for " + "floats, which is INFINITY and/or np.inf"}, + {"parse_card", (PyCFunction)PyFITS_parse_card, METH_VARARGS, + "parse_card\n\nparse the card to get the key name, value (as a string), " + "data type and comment."}, + {"get_keytype", (PyCFunction)PyFITS_get_keytype, METH_VARARGS, + "get_keytype\n\nparse the card to get the key type."}, + {"get_key_meta", (PyCFunction)PyFITS_get_key_meta, METH_VARARGS, + "get_key_meta\n\nparse the card to get key metadata (keyclass,dtype)."}, + {NULL} /* Sentinel */ +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_fitsio_wrap", /* m_name */ + "Defines the FITS class and some methods", /* m_doc */ + -1, /* m_size */ + fitstype_methods, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; +#endif + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC +#if PY_MAJOR_VERSION >= 3 +PyInit__fitsio_wrap(void) +#else +init_fitsio_wrap(void) +#endif +{ + PyObject *m; + + PyFITSType.tp_new = PyType_GenericNew; + +#if PY_MAJOR_VERSION >= 3 + if (PyType_Ready(&PyFITSType) < 0) { + return NULL; + } + m = PyModule_Create(&moduledef); + if (m == NULL) { + return NULL; + } + +#else + if (PyType_Ready(&PyFITSType) < 0) { + return; + } + m = Py_InitModule3("_fitsio_wrap", fitstype_methods, + "Define FITS type and methods."); + if (m == NULL) { + return; + } +#endif + + Py_INCREF(&PyFITSType); + PyModule_AddObject(m, "FITS", (PyObject *)&PyFITSType); + + import_array(); +#if PY_MAJOR_VERSION >= 3 + return m; +#endif +} diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py new file mode 100644 index 0000000..149eb46 --- /dev/null +++ b/fitsio/fitslib.py @@ -0,0 +1,2119 @@ +""" +fitslib, part of the fitsio package. + +See the main docs at https://github.com/esheldon/fitsio + + Copyright (C) 2011 Erin Sheldon, BNL. erin dot sheldon at gmail dot com + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" + +from __future__ import with_statement, print_function +import os +import numpy + +from . import _fitsio_wrap +from .util import ( + IS_PY3, + mks, + array_to_native, + isstring, + copy_if_needed, + _nonfinite_as_cfitsio_floating_null_value, +) +from .header import FITSHDR +from .hdu import ( + ANY_HDU, + IMAGE_HDU, + BINARY_TBL, + ASCII_TBL, + ImageHDU, + AsciiTableHDU, + TableHDU, + _table_npy2fits_form, + _npy2fits, + _hdu_type_map, +) + +from .fits_exceptions import FITSFormatError + +# for python3 compat +if IS_PY3: + xrange = range + + +READONLY = 0 +READWRITE = 1 + +# this constant is used to indicate +# that an option is not set in Python +# and instead the setting for that option +# is delegated to the C code in cfitsio +NOT_SET = "NOT_SET" + +NOCOMPRESS = 0 +RICE_1 = 11 +GZIP_1 = 21 +GZIP_2 = 22 +PLIO_1 = 31 +HCOMPRESS_1 = 41 + +NO_DITHER = -1 +SUBTRACTIVE_DITHER_1 = 1 +SUBTRACTIVE_DITHER_2 = 2 + + +def read( + filename, + ext=None, + extver=None, + columns=None, + rows=None, + header=False, + case_sensitive=False, + upper=False, + lower=False, + vstorage='fixed', + verbose=False, + trim_strings=False, + **keys, +): + """ + Convenience function to read data from the specified FITS HDU + + By default, all data are read. For tables, send columns= and rows= to + select subsets of the data. Table data are read into a recarray; use a + FITS object and read_column() to get a single column as an ordinary array. + For images, create a FITS object and use slice notation to read subsets. + + Under the hood, a FITS object is constructed and data are read using + an associated FITSHDU object. + + parameters + ---------- + filename: string + A filename. + ext: number or string, optional + The extension. Either the numerical extension from zero + or a string extension name. If not sent, data is read from + the first HDU that has data. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). These + extensions can optionally specify an EXTVER version number in the + header. Send extver= to select a particular version. If extver is not + sent, the first one will be selected. If ext is an integer, the extver + is ignored. + columns: list or array, optional + An optional set of columns to read from table HDUs. Default is to + read all. Can be string or number. + rows: optional + An optional list of rows to read from table HDUS. Default is to + read all. + header: bool, optional + If True, read the FITS header and return a tuple (data,header) + Default is False. + case_sensitive: bool, optional + Match column names and extension names with case-sensitivity. Default + is False. + lower: bool, optional + If True, force all columns names to lower case in output. Default is + False. + upper: bool, optional + If True, force all columns names to upper case in output. Default is + False. + vstorage: string, optional + Set the default method to store variable length columns. Can be + 'fixed' or 'object'. See docs on fitsio.FITS for details. Default is + 'fixed'. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + verbose: bool, optional + If True, print more info when doing various FITS operations. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + kwargs = { + 'lower': lower, + 'upper': upper, + 'vstorage': vstorage, + 'case_sensitive': case_sensitive, + 'verbose': verbose, + 'trim_strings': trim_strings, + } + + read_kwargs = {} + if columns is not None: + read_kwargs['columns'] = columns + if rows is not None: + read_kwargs['rows'] = rows + + with FITS(filename, **kwargs) as fits: + if ext is None: + for i in xrange(len(fits)): + if fits[i].has_data(): + ext = i + break + if ext is None: + raise IOError("No extensions have data") + + item = _make_item(ext, extver=extver) + + data = fits[item].read(**read_kwargs) + if header: + h = fits[item].read_header() + return data, h + else: + return data + + +def read_header(filename, ext=0, extver=None, case_sensitive=False, **keys): + """ + Convenience function to read the header from the specified FITS HDU + + The FITSHDR allows access to the values and comments by name and + number. + + parameters + ---------- + filename: string + A filename. + ext: number or string, optional + The extension. Either the numerical extension from zero + or a string extension name. Default read primary header. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). These + extensions can optionally specify an EXTVER version number in the + header. Send extver= to select a particular version. If extver is not + sent, the first one will be selected. If ext is an integer, the extver + is ignored. + case_sensitive: bool, optional + Match extension names with case-sensitivity. Default is False. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + filename = extract_filename(filename) + + dont_create = 0 + try: + hdunum = ext + 1 + except TypeError: + hdunum = None + + _fits = _fitsio_wrap.FITS(filename, READONLY, dont_create) + + if hdunum is None: + extname = mks(ext) + if extver is None: + extver_num = 0 + else: + extver_num = extver + + if not case_sensitive: + # the builtin movnam_hdu is not case sensitive + hdunum = _fits.movnam_hdu(ANY_HDU, extname, extver_num) + else: + # for case sensitivity we'll need to run through + # all the hdus + found = False + current_ext = 0 + while True: + hdunum = current_ext + 1 + try: + hdu_type = _fits.movabs_hdu(hdunum) # noqa - not used + name, vers = _fits.get_hdu_name_version(hdunum) + if name == extname: + if extver is None: + # take the first match + found = True + break + else: + if extver_num == vers: + found = True + break + except OSError: + break + + current_ext += 1 + + if not found: + raise IOError( + 'hdu not found: %s (extver %s)' % (extname, extver) + ) + + return FITSHDR(_fits.read_header(hdunum)) + + +def read_scamp_head(fname, header=None): + """ + read a SCAMP .head file as a fits header FITSHDR object + + parameters + ---------- + fname: string + The path to the SCAMP .head file + + header: FITSHDR, optional + Optionally combine the header with the input one. The input can + be any object convertable to a FITSHDR object + + returns + ------- + header: FITSHDR + A fits header object of type FITSHDR + """ + + with open(fname) as fobj: + lines = fobj.readlines() + + lines = [line.strip() for line in lines if line[0:3] != 'END'] + + # if header is None an empty FITSHDR is created + hdr = FITSHDR(header) + + for line in lines: + hdr.add_record(line) + + return hdr + + +def _make_item(ext, extver=None): + if extver is not None: + # e + item = (ext, extver) + else: + item = ext + + return item + + +class _DocStringFormatter(dict): + """A class to manager docstring snippets. + + This is a simpler version of the _SnippetManager + from proplot/ultraplot + """ + + def __call__(self, func_or_meth): + import inspect + + func_or_meth.__doc__ = inspect.getdoc(func_or_meth) + if func_or_meth.__doc__: + func_or_meth.__doc__ %= self + + return func_or_meth + + def __setitem__(self, key, value): + super().__setitem__(key, value.strip("\n")) + + +_doc_string_formatter = _DocStringFormatter() +_doc_string_formatter["compression_docs"] = """\ +compress: string, optional + A string representing the compression algorithm for images. + Default of fitsio.NOT_SET defers the setting to the default + of the cfitsio library (no compression) or to the value set + in the FITS file extended filename syntax (e.g., + `myfile.fits[compress G]`). + For no compression, pass None, fitsio.NOCOMPRESS, or 0. + For compression, can be one of + 'RICE' + 'GZIP' + 'GZIP_2' + 'PLIO' (no unsigned or negative integers) + 'HCOMPRESS' + (case-insensitive). See the cfitsio manual for details. +tile_dims: tuple of ints, optional + The size of the tiles used to compress images, specified in + Fortran/column-major order (e.g., `(Y_SIZE, X_SIZE)`). Default of + fitsio.NOT_SET defers the setting to the cfitsio/fpack (row-by-row) + or to the value set in the FITS file extended filename syntax (e.g., + `myfile.fits[compress G 100,100]`). The value None behaves the same as + fitsio.NOT_SET +qlevel: float, optional + Quantization level for floating point data. Lower generally result in + more compression, we recommend one reads the FITS standard or cfitsio + manual to fully understand the effects of quantization. None or 0 + means no quantization, and for gzip also implies lossless. Default of + fitsio.NOT_SET defers to the cfitsio/fpack default (usually 4.0) or + to the value set in the FITS file extended filename syntax (e.g., + `myfile.fits[compress G; q 10.0]`). +qmethod: string or int + The quantization method as string or integer. + 'NO_DITHER' or fitsio.NO_DITHER (-1) + No dithering is performed + 'SUBTRACTIVE_DITHER_1' or fitsio.SUBTRACTIVE_DITHER_1 (1) + Standard dithering + 'SUBTRACTIVE_DITHER_2' or fitsio.SUBTRACTIVE_DITHER_2 (2) + Preserves zeros + Default of fitsio.NOT_SET defers to the cfitsio/fpack default ( + 'SUBTRACTIVE_DITHER_1') or to the value set in the FITS file + extended filename syntax (e.g.,. `myfile.fits[compress R; qz]`). +dither_seed: int or None, optional + Seed for the subtractive dither. Seeding makes the lossy compression + reproducible. Allowed values are + fitsio.NOT_SET + defer the setting to the cfitsio/fpack library default + (system clock) + None or 0 or 'clock': + do not set the seed explicitly, use the system clock + negative or 'checksum': + set the seed based on the data checksum + 1-10_000: + use the input seed +hcomp_scale: float, optional + Scale value for HCOMPRESS, 0.0 means lossless compression. Default + of fitsio.NOT_SET defers to the cfitsio/fpack default (1.0) or + to the value set in the FITS file extended filename syntax (e.g., + `myfile.fits[compress H 10,10; s 10]`). +hcomp_smooth: bool, optional + If True, apply smoothing when decompressing, otherwise if False do not. + Default of fitsio.NOT_SET defers to the cfitsio/fpack default (False) or + to the value set in the FITS file extended filename syntax (e.g., + `myfile.fits[compress HS 10,10; s 10]`). + +**If the FITS file uses the extended filename syntax to set any compression +paraneters (e.g. `myfile.fits[compress R]`), then the those parameters +are treated as immutable defaults. If you set any of the Python keyword +compression parameters (i.e., compress, tile_dims, qlevel, qmethod, +hcomp_scale, hcomp_smooth), then the code will raise a ValueError. However, +the dither_seed can be set since it is not possible to set it via the +extended filename syntax.** +""" + + +@_doc_string_formatter +def write( + filename, + data, + extname=None, + extver=None, + header=None, + clobber=False, + ignore_empty=False, + units=None, + table_type='binary', + names=None, + write_bitcols=False, + compress=NOT_SET, + tile_dims=NOT_SET, + qlevel=NOT_SET, + qmethod=NOT_SET, + dither_seed=NOT_SET, + hcomp_scale=NOT_SET, + hcomp_smooth=NOT_SET, + **keys, +): + """ + Convenience function to create a new HDU and write the data. + + Under the hood, a FITS object is constructed. If you want to append rows + to an existing HDU, or modify data in an HDU, please construct a FITS + object. + + parameters + ---------- + filename: string + A filename. + data: numpy.ndarray or recarray + Either a normal n-dimensional array or a recarray. Images are written + to a new IMAGE_HDU and recarrays are written to BINARY_TBl or + ASCII_TBL hdus. + extname: string, optional + An optional name for the new header unit. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). + These extensions can optionally specify an EXTVER version number in + the header. Send extver= to set a particular version, which will + be represented in the header with keyname EXTVER. The extver must + be an integer > 0. If extver is not sent, the first one will be + selected. If ext is an integer, the extver is ignored. + header: FITSHDR, list, dict, optional + A set of header keys to write. The keys are written before the data + is written to the table, preventing a resizing of the table area. + + Can be one of these: + - FITSHDR object + - list of dictionaries containing 'name','value' and optionally + a 'comment' field; the order is preserved. + - a dictionary of keyword-value pairs; no comments are written + in this case, and the order is arbitrary. + Note required keywords such as NAXIS, XTENSION, etc are cleaed out. + clobber: bool, optional + If True, overwrite any existing file. Default is to append + a new extension on existing files. + ignore_empty: bool, optional + Default False. Unless set to True, only allow + empty HDUs in the zero extension. + + table-only keywords + ------------------- + units: list + A list of strings representing units for each column. + table_type: string, optional + Either 'binary' or 'ascii', default 'binary' + Matching is case-insensitive + write_bitcols: bool, optional + Write boolean arrays in the FITS bitcols format, default False + names: list, optional + If data is a list of arrays, you must send `names` as a list + of names or column numbers. + + image-only keywords + ------------------- + %(compression_docs)s + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + kwargs = {'clobber': clobber, 'ignore_empty': ignore_empty} + with FITS(filename, 'rw', **kwargs) as fits: + fits.write( + data, + table_type=table_type, + units=units, + extname=extname, + extver=extver, + header=header, + names=names, + write_bitcols=write_bitcols, + compress=compress, + tile_dims=tile_dims, + qlevel=qlevel, + qmethod=qmethod, + dither_seed=dither_seed, + hcomp_scale=hcomp_scale, + hcomp_smooth=hcomp_smooth, + ) + + +class FITS(object): + """ + A class to read and write FITS images and tables. + + This class uses the cfitsio library for almost all relevant work. + + parameters + ---------- + filename: string + The filename to open. + mode: int/string, optional + The mode, either a string or integer. + For reading only + 'r' or 0 + For reading and writing + 'rw' or 1 + You can also use fitsio.READONLY and fitsio.READWRITE. + + Default is 'r' + clobber: bool, optional + If the mode is READWRITE, and clobber=True, then remove any existing + file before opening. + case_sensitive: bool, optional + Match column names and extension names with case-sensitivity. Default + is False. + lower: bool, optional + If True, force all columns names to lower case in output + upper: bool, optional + If True, force all columns names to upper case in output + vstorage: string, optional + A string describing how, by default, to store variable length columns + in the output array. This can be over-ridden when reading by using the + using vstorage keyword to the individual read methods. The options are + + 'fixed': Use a fixed length field in the array, with + dimensions equal to the max possible size for column. + Arrays are padded with zeros. + 'object': Use an object for the field in the array. + Each element will then be an array of the right type, + but only using the memory needed to hold that element. + + Default is 'fixed'. The rationale is that this is the option + of 'least surprise' + iter_row_buffer: integer + Number of rows to buffer when iterating over table HDUs. + Default is 1. + ignore_empty: bool, optional + Default False. Unless set to True, only allow + empty HDUs in the zero extension. + verbose: bool, optional + If True, print more info when doing various FITS operations. + + See the docs at https://github.com/esheldon/fitsio + """ + + def __init__( + self, + filename, + mode='r', + lower=False, + upper=False, + trim_strings=False, + vstorage='fixed', + case_sensitive=False, + iter_row_buffer=1, + write_bitcols=False, + ignore_empty=False, + verbose=False, + clobber=False, + **keys, + ): + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + self.lower = lower + self.upper = upper + self.trim_strings = trim_strings + self.vstorage = vstorage + self.case_sensitive = case_sensitive + self.iter_row_buffer = iter_row_buffer + self.write_bitcols = write_bitcols + filename = extract_filename(filename) + self._filename = filename + + # self.mode=keys.get('mode','r') + self.mode = mode + self.ignore_empty = ignore_empty + + self.verbose = verbose + + if self.mode not in _int_modemap: + raise IOError( + "mode should be one of 'r', 'rw', READONLY,READWRITE" + ) + + self.charmode = _char_modemap[self.mode] + self.intmode = _int_modemap[self.mode] + + # Will not test existence when reading, let cfitsio + # do the test and report an error. This allows opening + # urls etc. + create = 0 + if self.mode in [READWRITE, 'rw']: + if clobber: + create = 1 + if filename[0] != '!': + filename = '!' + filename + else: + if os.path.exists(filename): + create = 0 + else: + create = 1 + + self._did_create = create == 1 + self._FITS = _fitsio_wrap.FITS(filename, self.intmode, create) + + def close(self): + """ + Close the fits file and set relevant metadata to None + """ + if hasattr(self, '_FITS'): + if self._FITS is not None: + self._FITS.close() + self._FITS = None + self._filename = None + self.mode = None + self.charmode = None + self.intmode = None + self.hdu_list = None + self.hdu_map = None + + def movabs_ext(self, ext): + """ + Move to the indicated zero-offset extension. + + In general, it is not necessary to use this method explicitly. + """ + return self.movabs_hdu(ext + 1) + + def movabs_hdu(self, hdunum): + """ + Move to the indicated one-offset hdu number. + + In general, it is not necessary to use this method explicitly. + """ + + format_err = False + + try: + hdu_type = self._FITS.movabs_hdu(hdunum) + except IOError as err: + # to support python 2 we can't use exception chaining. + # do this to avoid "During handling of the above exception, another + # exception occurred:" + serr = str(err) + if 'first keyword not XTENSION' in serr: + format_err = True + else: + raise + + if format_err: + raise FITSFormatError(serr) + + return hdu_type + + def movnam_ext(self, extname, hdutype=ANY_HDU, extver=0): + """ + Move to the indicated extension by name + + In general, it is not necessary to use this method explicitly. + + returns the zero-offset extension number + """ + extname = mks(extname) + hdu = self._FITS.movnam_hdu(hdutype, extname, extver) + return hdu - 1 + + def movnam_hdu(self, extname, hdutype=ANY_HDU, extver=0): + """ + Move to the indicated HDU by name + + In general, it is not necessary to use this method explicitly. + + returns the one-offset extension number + """ + format_err = False + + extname = mks(extname) + try: + hdu = self._FITS.movnam_hdu(hdutype, extname, extver) + except IOError as err: + # to support python 2 we can't use exception chaining. + # do this to avoid "During handling of the above exception, another + # exception occurred:" + serr = str(err) + if 'first keyword not XTENSION' in serr: + format_err = True + else: + raise + + if format_err: + raise FITSFormatError(serr) + + return hdu + + def reopen(self): + """ + close and reopen the fits file with the same mode + """ + # We cannot open mem:// memory files as existing files + # (i.e., last argument of _fitsio_wrap.FITS equal to 0). + # If we open in mode 1, we will delete all of the existing data + # in the mem:// file. So we skip the close+reopen cycle for + # mem:// files. We always update the hdu list and this appears + # to be important. + if not self._filename.startswith("mem://"): + self._FITS.close() + del self._FITS + self._FITS = _fitsio_wrap.FITS(self._filename, self.intmode, 0) + self.update_hdu_list() + + @_doc_string_formatter + def write( + self, + data, + units=None, + extname=None, + extver=None, + compress=NOT_SET, + tile_dims=NOT_SET, + qlevel=NOT_SET, + qmethod=NOT_SET, + dither_seed=NOT_SET, + hcomp_scale=NOT_SET, + hcomp_smooth=NOT_SET, + header=None, + names=None, + table_type='binary', + write_bitcols=False, + **keys, + ): + """ + Write the data to a new HDU. + + This method is a wrapper. If this is an IMAGE_HDU, write_image is + called, otherwise write_table is called. + + parameters + ---------- + data: ndarray + An n-dimensional image or an array with fields. + extname: string, optional + An optional extension name. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). + These extensions can optionally specify an EXTVER version number in + the header. Send extver= to set a particular version, which will + be represented in the header with keyname EXTVER. The extver must + be an integer > 0. If extver is not sent, the first one will be + selected. If ext is an integer, the extver is ignored. + header: FITSHDR, list, dict, optional + A set of header keys to write. Can be one of these: + - FITSHDR object + - list of dictionaries containing 'name','value' and optionally + a 'comment' field; the order is preserved. + - a dictionary of keyword-value pairs; no comments are written + in this case, and the order is arbitrary. + Note required keywords such as NAXIS, XTENSION, etc are cleaed out. + + image-only keywords + ------------------- + %(compression_docs)s + + table-only keywords + ------------------- + units: list/dec, optional: + A list of strings with units for each column. + table_type: string, optional + Either 'binary' or 'ascii', default 'binary' + Matching is case-insensitive + write_bitcols: bool, optional + Write boolean arrays in the FITS bitcols format, default False + names: list, optional + If data is a list of arrays, you must send `names` as a list + of names or column numbers. + + restrictions + ------------ + The File must be opened READWRITE + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + isimage = False + if data is None: + isimage = True + elif isinstance(data, numpy.ndarray): + if data.dtype.fields == None: # noqa - probably should be is None + isimage = True + + if isimage: + self.write_image( + data, + extname=extname, + extver=extver, + compress=compress, + tile_dims=tile_dims, + qlevel=qlevel, + qmethod=qmethod, + dither_seed=dither_seed, + hcomp_scale=hcomp_scale, + hcomp_smooth=hcomp_smooth, + header=header, + ) + else: + self.write_table( + data, + units=units, + extname=extname, + extver=extver, + header=header, + names=names, + table_type=table_type, + write_bitcols=write_bitcols, + ) + + @_doc_string_formatter + def write_image( + self, + img, + extname=None, + extver=None, + compress=NOT_SET, + tile_dims=NOT_SET, + qlevel=NOT_SET, + qmethod=NOT_SET, + dither_seed=NOT_SET, + hcomp_scale=NOT_SET, + hcomp_smooth=NOT_SET, + header=None, + ): + """ + Create a new image extension and write the data. + + parameters + ---------- + img: ndarray + An n-dimensional image. + extname: string, optional + An optional extension name. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). + These extensions can optionally specify an EXTVER version number in + the header. Send extver= to set a particular version, which will + be represented in the header with keyname EXTVER. The extver must + be an integer > 0. If extver is not sent, the first one will be + selected. If ext is an integer, the extver is ignored. + header: FITSHDR, list, dict, optional + A set of header keys to write. Can be one of these: + - FITSHDR object + - list of dictionaries containing 'name','value' and optionally + a 'comment' field; the order is preserved. + - a dictionary of keyword-value pairs; no comments are written + in this case, and the order is arbitrary. + Note required keywords such as NAXIS, XTENSION, etc are cleaed out. + %(compression_docs)s + + restrictions + ------------ + The File must be opened READWRITE + """ + + self.create_image_hdu( + img, + header=header, + extname=extname, + extver=extver, + compress=compress, + tile_dims=tile_dims, + qlevel=qlevel, + qmethod=qmethod, + dither_seed=dither_seed, + hcomp_scale=hcomp_scale, + hcomp_smooth=hcomp_smooth, + ) + + if header is not None: + self[-1].write_keys(header) + + # if img is not None: + # self[-1].write(img) + + @_doc_string_formatter + def create_image_hdu( + self, + img=None, + dims=None, + dtype=None, + extname=None, + extver=None, + compress=NOT_SET, + tile_dims=NOT_SET, + qlevel=NOT_SET, + qmethod=NOT_SET, + dither_seed=NOT_SET, + hcomp_scale=NOT_SET, + hcomp_smooth=NOT_SET, + header=None, + ): + """ + Create a new, empty image HDU and reload the hdu list. Either + create from an input image or from input dims and dtype + + fits.create_image_hdu(image, ...) + fits.create_image_hdu(dims=dims, dtype=dtype) + + If an image is sent, the data are also written. + + You can write data into the new extension using + fits[extension].write(image) + + Alternatively you can skip calling this function and instead just use + + fits.write(image) + or + fits.write_image(image) + + which will create the new image extension for you with the appropriate + structure, and write the data. + + parameters + ---------- + img: ndarray, optional + An image with which to determine the properties of the HDU. The + data will be written. + dims: sequence, optional + A sequence describing the dimensions of the image to be created + on disk. You must also send a dtype= + dtype: numpy data type + When sending dims= also send the data type. Can be of the + various numpy data type declaration styles, e.g. 'f8', + numpy.float64. + extname: string, optional + An optional extension name. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). + These extensions can optionally specify an EXTVER version number in + the header. Send extver= to set a particular version, which will + be represented in the header with keyname EXTVER. The extver must + be an integer > 0. If extver is not sent, the first one will be + selected. If ext is an integer, the extver is ignored. + header: FITSHDR, list, dict, optional + This is only used to determine how many slots to reserve for + header keywords + %(compression_docs)s + + restrictions + ------------ + The File must be opened READWRITE + """ + + if (img is not None) or (img is None and dims is None): + from_image = True + elif dims is not None: + from_image = False + + if from_image: + img2send = img + if img is not None: + dims = img.shape + dtstr = img.dtype.descr[0][1][1:] + if img.size == 0: + raise ValueError("data must have at least 1 row") + + # data must be c-contiguous and native byte order + if not img.flags['C_CONTIGUOUS']: + # this always makes a copy + img2send = numpy.ascontiguousarray(img) + img2send = array_to_native(img2send, inplace=True) + else: + img2send = array_to_native(img, inplace=False) + + if IS_PY3 and img2send.dtype.char == 'U': + # for python3, we convert unicode to ascii + # this will error if the character is not in ascii + img2send = img2send.astype('S', copy=copy_if_needed) + + else: + self._ensure_empty_image_ok() + compress = None + tile_dims = None + + # we get dims from the input image + dims2send = None + else: + # img was None and dims was sent + if dtype is None: + raise ValueError("send dtype= with dims=") + + # this must work! + dtype = numpy.dtype(dtype) + dtstr = dtype.descr[0][1][1:] + # use the example image to build the type in C + img2send = numpy.zeros(1, dtype=dtype) + + # sending an array simplifies access + dims2send = numpy.array(dims, dtype='i8', ndmin=1) + + if img2send is not None: + if img2send.dtype.fields is not None: + raise ValueError( + "got record data type, expected regular ndarray" + ) + + if extname is None: + # will be ignored + extname = "" + else: + if not isstring(extname): + raise ValueError("extension name must be a string") + extname = mks(extname) + + if extname is not None and extver is not None: + extver = check_extver(extver) + + if extver is None: + # will be ignored + extver = 0 + + # if the file is using the extended filename syntax for + # compression, then we ignore any input compression params + # and raise if they are not fitsio.NOT_SET + # we do allow the dither_seed since there is no way to set + # this via the extended filename syntax + if "[compress" in self._filename.lower(): + if ( + compress != NOT_SET + or (not (isinstance(tile_dims, str) and tile_dims == NOT_SET)) + or qlevel != NOT_SET + or qmethod != NOT_SET + or hcomp_scale != NOT_SET + or hcomp_smooth != NOT_SET + ): + raise ValueError( + "You cannot override the compression parameters " + "from Python for " + "FITS files that use the extend filename syntax " + "(e.g., `myfile.fits[compress]`) for compression." + ) + + # For FITS file using the extend filename syntax for + # compression, we do not allow overrides from Python. + # The value None is equivalent to "not set" at the + # C level and will ensure no overrides are done. + comptype = None + qmethod = None + tile_dims = None + qlevel = None + hcs = None + hcomp_scale = None + else: + comptype = get_compress_type(compress) + if img2send is not None: + check_comptype_img(comptype, dtstr) + + qmethod = get_qmethod(qmethod) + tile_dims = get_tile_dims(tile_dims, dims) + + if qlevel == NOT_SET: + # in this case, we pass None since in the C layer, + # the value None means not set. + qlevel = None + elif qlevel is None: + # in the Python layer qlevel being None means no quantization. + # thus we pass 0.0 since + # it is the sentinel value for "no quantization" in cfitsio + qlevel = 0.0 + else: + qlevel = float(qlevel) + + if hcomp_smooth == NOT_SET: + hcs = None + else: + if hcomp_smooth: + hcs = 1 + else: + hcs = 0 + + if hcomp_scale == NOT_SET: + hcomp_scale = None + + # we always allow dither seed to be set + dither_seed = get_dither_seed(dither_seed) + + if header is not None: + nkeys = len(header) + else: + nkeys = 0 + + if comptype != NOT_SET or "[compress" in self._filename.lower(): + hdu_is_compressed = True + else: + hdu_is_compressed = False + + with _nonfinite_as_cfitsio_floating_null_value( + img2send, hdu_is_compressed + ) as img2send_any_nan: + img2send, any_nan = img2send_any_nan + self._FITS.create_image_hdu( + img2send, + nkeys, + dims=dims2send, + comptype=comptype, + tile_dims=tile_dims, + qlevel=qlevel, + qmethod=qmethod, + dither_seed=dither_seed, + hcomp_scale=hcomp_scale, + hcomp_smooth=hcs, + extname=extname, + extver=extver, + any_nan=1 if any_nan else 0, + ) + + self.update_hdu_list(rebuild=False) + + def _ensure_empty_image_ok(self): + """ + If ignore_empty was not set to True, we only allow empty HDU for first + HDU and if there is no data there already + """ + if self.ignore_empty: + return + + if len(self) > 1: + raise RuntimeError( + "Cannot write None image at extension %d" % len(self) + ) + if 'ndims' in self[0]._info: + raise RuntimeError( + "Can only write None images to extension zero, " + "which already exists" + ) + + def write_table( + self, + data, + table_type='binary', + names=None, + formats=None, + units=None, + extname=None, + extver=None, + header=None, + write_bitcols=False, + ): + """ + Create a new table extension and write the data. + + The table definition is taken from the fields in the input array. If + you want to append new rows to the table, access the HDU directly and + use the write() function, e.g. + + fits[extension].append(data) + + parameters + ---------- + data: recarray + A numpy array with fields. The table definition will be + determined from this array. + table_type: string, optional + Either 'binary' or 'ascii', default 'binary' + Matching is case-insensitive + extname: string, optional + An optional string for the extension name. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). + These extensions can optionally specify an EXTVER version number in + the header. Send extver= to set a particular version, which will + be represented in the header with keyname EXTVER. The extver must + be an integer > 0. If extver is not sent, the first one will be + selected. If ext is an integer, the extver is ignored. + units: list/dec, optional: + A list of strings with units for each column. + header: FITSHDR, list, dict, optional + A set of header keys to write. The keys are written before the data + is written to the table, preventing a resizing of the table area. + + Can be one of these: + - FITSHDR object + - list of dictionaries containing 'name','value' and optionally + a 'comment' field; the order is preserved. + - a dictionary of keyword-value pairs; no comments are written + in this case, and the order is arbitrary. + Note required keywords such as NAXIS, XTENSION, etc are cleaed out. + write_bitcols: boolean, optional + Write boolean arrays in the FITS bitcols format, default False + + restrictions + ------------ + The File must be opened READWRITE + """ + + """ + if data.dtype.fields == None: + raise ValueError("data must have fields") + if data.size == 0: + raise ValueError("data must have at least 1 row") + """ + + self.create_table_hdu( + data=data, + header=header, + names=names, + units=units, + extname=extname, + extver=extver, + table_type=table_type, + write_bitcols=write_bitcols, + ) + + if header is not None: + self[-1].write_keys(header) + + self[-1].write(data, names=names) + + def read_raw(self): + """ + Reads the raw FITS file contents, returning a Python string. + """ + return self._FITS.read_raw() + + def create_table_hdu( + self, + data=None, + dtype=None, + header=None, + names=None, + formats=None, + units=None, + dims=None, + extname=None, + extver=None, + table_type='binary', + write_bitcols=False, + ): + """ + Create a new, empty table extension and reload the hdu list. + + There are three ways to do it: + 1) send a numpy dtype, from which the formats in the fits file will + be determined. + 2) Send an array in data= keyword. this is required if you have + object fields for writing to variable length columns. + 3) send the names,formats and dims yourself + + You can then write data into the new extension using + fits[extension].write(array) + If you want to write to a single column + fits[extension].write_column(array) + But be careful as the other columns will be left zeroed. + + Often you will instead just use write_table to do this all + atomically. + + fits.write_table(recarray) + + write_table will create the new table extension for you with the + appropriate fields. + + parameters + ---------- + dtype: numpy dtype or descriptor, optional + If you have an array with fields, you can just send arr.dtype. You + can also use a list of tuples, e.g. [('x','f8'),('index','i4')] or + a dictionary representation. + data: a numpy array with fields, optional + or a dictionary + + An array or dict from which to determine the table definition. You + must use this instead of sending a descriptor if you have object + array fields, as this is the only way to determine the type and max + size. + + names: list of strings, optional + The list of field names + formats: list of strings, optional + The TFORM format strings for each field. + dims: list of strings, optional + An optional list of dimension strings for each field. Should + match the repeat count for the formats fields. Be careful of + the order since FITS is more like fortran. See the descr2tabledef + function. + + table_type: string, optional + Either 'binary' or 'ascii', default 'binary' + Matching is case-insensitive + units: list of strings, optional + An optional list of unit strings for each field. + extname: string, optional + An optional extension name. + extver: integer, optional + FITS allows multiple extensions to have the same name (extname). + These extensions can optionally specify an EXTVER version number in + the header. Send extver= to set a particular version, which will + be represented in the header with keyname EXTVER. The extver must + be an integer > 0. If extver is not sent, the first one will be + selected. If ext is an integer, the extver is ignored. + write_bitcols: bool, optional + Write boolean arrays in the FITS bitcols format, default False + + header: FITSHDR, list, dict, optional + This is only used to determine how many slots to reserve for + header keywords + + + restrictions + ------------ + The File must be opened READWRITE + """ + + # record this for the TableHDU object + write_bitcols = self.write_bitcols or write_bitcols + + # can leave as turn + table_type_int = _extract_table_type(table_type) + + if data is not None: + if isinstance(data, numpy.ndarray): + names, formats, dims = array2tabledef( + data, table_type=table_type, write_bitcols=write_bitcols + ) + elif isinstance(data, (list, dict)): + names, formats, dims = collection2tabledef( + data, + names=names, + table_type=table_type, + write_bitcols=write_bitcols, + ) + else: + raise ValueError( + "data must be an ndarray with fields or a dict" + ) + elif dtype is not None: + dtype = numpy.dtype(dtype) + names, formats, dims = descr2tabledef( + dtype.descr, + write_bitcols=write_bitcols, + table_type=table_type, + ) + else: + if names is None or formats is None: + raise ValueError( + "send either dtype=, data=, or names= and formats=" + ) + + if not isinstance(names, list) or not isinstance(formats, list): + raise ValueError("names and formats should be lists") + if len(names) != len(formats): + raise ValueError("names and formats must be same length") + + if dims is not None: + if not isinstance(dims, list): + raise ValueError("dims should be a list") + if len(dims) != len(names): + raise ValueError("names and dims must be same length") + + if units is not None: + if not isinstance(units, list): + raise ValueError("units should be a list") + if len(units) != len(names): + raise ValueError("names and units must be same length") + + if extname is None: + # will be ignored + extname = "" + else: + if not isstring(extname): + raise ValueError("extension name must be a string") + extname = mks(extname) + + if extname is not None and extver is not None: + extver = check_extver(extver) + if extver is None: + # will be ignored + extver = 0 + if extname is None: + # will be ignored + extname = "" + + if header is not None: + nkeys = len(header) + else: + nkeys = 0 + + # note we can create extname in the c code for tables, but not images + self._FITS.create_table_hdu( + table_type_int, + nkeys, + names, + formats, + tunit=units, + tdim=dims, + extname=extname, + extver=extver, + ) + + # don't rebuild the whole list unless this is the first hdu + # to be created + self.update_hdu_list(rebuild=False) + + def update_hdu_list(self, rebuild=True): + """ + Force an update of the entire HDU list + + Normally you don't need to call this method directly + + if rebuild is false or the hdu_list is not yet set, the list is + rebuilt from scratch + """ + if not hasattr(self, 'hdu_list'): + rebuild = True + + if rebuild: + self.hdu_list = [] + self.hdu_map = {} + + # we don't know how many hdus there are, so iterate + # until we can't open any more + ext_start = 0 + else: + # start from last + ext_start = len(self) + + ext = ext_start + while True: + try: + self._append_hdu_info(ext) + except IOError: + break + except RuntimeError: + break + + ext = ext + 1 + + def _append_hdu_info(self, ext): + """ + internal routine + + append info for indiciated extension + """ + + # raised IOError if not found + hdu_type = self.movabs_ext(ext) + + if hdu_type == IMAGE_HDU: + hdu = ImageHDU(self._FITS, ext) + elif hdu_type == BINARY_TBL: + hdu = TableHDU( + self._FITS, + ext, + lower=self.lower, + upper=self.upper, + trim_strings=self.trim_strings, + vstorage=self.vstorage, + case_sensitive=self.case_sensitive, + iter_row_buffer=self.iter_row_buffer, + write_bitcols=self.write_bitcols, + ) + elif hdu_type == ASCII_TBL: + hdu = AsciiTableHDU( + self._FITS, + ext, + lower=self.lower, + upper=self.upper, + trim_strings=self.trim_strings, + vstorage=self.vstorage, + case_sensitive=self.case_sensitive, + iter_row_buffer=self.iter_row_buffer, + write_bitcols=self.write_bitcols, + ) + else: + mess = "extension %s is of unknown type %s this is probably a bug" + mess = mess % (ext, hdu_type) + raise IOError(mess) + + self.hdu_list.append(hdu) + self.hdu_map[ext] = hdu + + extname = hdu.get_extname() + if not self.case_sensitive: + extname = extname.lower() + if extname != '': + # this will guarantee we default to *first* version, + # if version is not requested, using __getitem__ + if extname not in self.hdu_map: + self.hdu_map[extname] = hdu + + ver = hdu.get_extver() + if ver > 0: + key = '%s-%s' % (extname, ver) + self.hdu_map[key] = hdu + + def __iter__(self): + """ + begin iteration over HDUs + """ + if not hasattr(self, 'hdu_list'): + self.update_hdu_list() + self._iter_index = 0 + return self + + def next(self): + """ + Move to the next iteration + """ + if self._iter_index == len(self.hdu_list): + raise StopIteration + hdu = self.hdu_list[self._iter_index] + self._iter_index += 1 + return hdu + + __next__ = next + + def __len__(self): + """ + get the number of extensions + """ + if not hasattr(self, 'hdu_list'): + self.update_hdu_list() + return len(self.hdu_list) + + def _extract_item(self, item): + """ + utility function to extract an "item", meaning + a extension number,name plus version. + """ + ver = 0 + if isinstance(item, tuple): + ver_sent = True + nitem = len(item) + if nitem == 1: + ext = item[0] + elif nitem == 2: + ext, ver = item + else: + ver_sent = False + ext = item + return ext, ver, ver_sent + + def __getitem__(self, item): + """ + Get an hdu by number, name, and possibly version + """ + if not hasattr(self, 'hdu_list'): + if self._did_create: + # we created the file and haven't written anything yet + raise ValueError("Requested hdu '%s' not present" % item) + + self.update_hdu_list() + + if len(self) == 0: + raise ValueError("Requested hdu '%s' not present" % item) + + ext, ver, ver_sent = self._extract_item(item) + + try: + # if it is an int + hdu = self.hdu_list[ext] + except Exception: + # might be a string + ext = mks(ext) + if not self.case_sensitive: + mess = '(case insensitive)' + ext = ext.lower() + else: + mess = '(case sensitive)' + + if ver > 0: + key = '%s-%s' % (ext, ver) + if key not in self.hdu_map: + raise IOError( + "extension not found: %s, " + "version %s %s" % (ext, ver, mess) + ) + hdu = self.hdu_map[key] + else: + if ext not in self.hdu_map: + raise IOError("extension not found: %s %s" % (ext, mess)) + hdu = self.hdu_map[ext] + + return hdu + + def __contains__(self, item): + """ + tell whether specified extension exists, possibly + with version sent as well + """ + try: + hdu = self[item] # noqa + return True + except Exception: + return False + + def __repr__(self): + """ + Text representation of some fits file metadata + """ + spacing = ' ' * 2 + rep = [''] + rep.append("%sfile: %s" % (spacing, self._filename)) + rep.append("%smode: %s" % (spacing, _modeprint_map[self.intmode])) + + rep.append('%sextnum %-15s %s' % (spacing, "hdutype", "hduname[v]")) + + if not hasattr(self, 'hdu_list'): + if not self._did_create: + # we expect some stuff + self.update_hdu_list() + + for i, hdu in enumerate(self.hdu_list): + t = hdu._info['hdutype'] + name = hdu.get_extname() + if name != '': + ver = hdu.get_extver() + if ver != 0: + name = '%s[%s]' % (name, ver) + + rep.append( + "%s%-6d %-15s %s" % (spacing, i, _hdu_type_map[t], name) + ) + + rep = '\n'.join(rep) + return rep + + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + self.close() + + +def check_extver(extver): + if extver is None: + return 0 + extver = int(extver) + if extver <= 0: + raise ValueError("extver must be > 0") + return extver + + +def extract_filename(filename): + filename = mks(filename) + filename = filename.strip() + if filename[0] == "!": + filename = filename[1:] + filename = os.path.expandvars(filename) + filename = os.path.expanduser(filename) + return filename + + +def array2tabledef(data, table_type='binary', write_bitcols=False): + """ + Similar to descr2tabledef but if there are object columns a type + and max length will be extracted and used for the tabledef + """ + is_ascii = table_type == 'ascii' + + if data.dtype.fields is None: + raise ValueError("data must have fields") + names = [] + names_nocase = {} + formats = [] + dims = [] + + descr = data.dtype.descr + for d in descr: + # these have the form ' 1: + send_dt = list(dt) + [this_data.shape[1:]] + _, form, dim = _npy2fits( + send_dt, table_type=table_type, write_bitcols=write_bitcols + ) + + formats.append(form) + dims.append(dim) + + return names, formats, dims + + +def descr2tabledef(descr, table_type='binary', write_bitcols=False): + """ + Create a FITS table def from the input numpy descriptor. + + parameters + ---------- + descr: list + A numpy recarray type descriptor array.dtype.descr + + returns + ------- + names, formats, dims: tuple of lists + These are the ttyp, tform and tdim header entries + for each field. dim entries may be None + """ + names = [] + formats = [] + dims = [] + + for d in descr: + """ + npy_dtype = d[1][1:] + if is_ascii and npy_dtype in ['u1','i1']: + raise ValueError("1-byte integers are not supported for " + "ascii tables") + """ + + if d[1][1] == 'O': + raise ValueError( + 'cannot automatically declare a var column without ' + 'some data to determine max len' + ) + + name, form, dim = _npy2fits( + d, table_type=table_type, write_bitcols=write_bitcols + ) + + if name == '': + raise ValueError("field name is an empty string") + + """ + if is_ascii: + if dim is not None: + raise ValueError("array columns are not supported " + "for ascii tables") + """ + + names.append(name) + formats.append(form) + dims.append(dim) + + return names, formats, dims + + +def npy_obj2fits(data, name=None): + # this will be a variable length column 1Pt(len) where t is the + # type and len is max length. Each element must be convertible to + # the same type as the first + + if name is None: + d = data.dtype.descr + first = data[0] + else: + d = data[name].dtype.descr # noqa - not used + first = data[name][0] + + # note numpy._string is an instance of str in python2, bytes + # in python3 + if isinstance(first, str) or (IS_PY3 and isinstance(first, bytes)): + if IS_PY3: + if isinstance(first, str): + fits_dtype = _table_npy2fits_form['U'] + else: + fits_dtype = _table_npy2fits_form['S'] + else: + fits_dtype = _table_npy2fits_form['S'] + else: + arr0 = numpy.array(first, copy=copy_if_needed) + dtype0 = arr0.dtype + npy_dtype = dtype0.descr[0][1][1:] + if npy_dtype[0] == 'S' or npy_dtype[0] == 'U': + raise ValueError( + "Field '%s' is an arrays of strings, this is " + "not allowed in variable length columns" % name + ) + if npy_dtype not in _table_npy2fits_form: + raise ValueError( + "Field '%s' has unsupported type '%s'" % (name, npy_dtype) + ) + fits_dtype = _table_npy2fits_form[npy_dtype] + + # Q uses 64-bit addressing, should try at some point but the cfitsio manual + # says it is experimental + # form = '1Q%s' % fits_dtype + form = '1P%s' % fits_dtype + dim = None + + return form, dim + + +def get_tile_dims(tile_dims, imshape): + """ + Just make sure the tile dims has the appropriate number of dimensions + """ + if tile_dims is None or ( + isinstance(tile_dims, str) and tile_dims == NOT_SET + ): + td = None + else: + td = numpy.array(tile_dims, dtype='i8') + nd = len(imshape) + if td.size != nd: + msg = "expected tile_dims to have %d dims, got %d" % (td.size, nd) + raise ValueError(msg) + + return td + + +def get_compress_type(compress): + if compress == NOT_SET: + return None + + if compress is not None: + compress = str(compress).upper() + if compress not in _compress_map: + raise ValueError( + "compress must be one of %s" % list(_compress_map.keys()) + ) + return _compress_map[compress] + + +def get_qmethod(qmethod): + if qmethod == NOT_SET: + return None + + if qmethod not in _qmethod_map: + if isinstance(qmethod, str): + qmethod = qmethod.upper() + elif isinstance(qmethod, bytes): + # in py27, bytes are str, so we can safely assume + # py3 here + qmethod = str(qmethod, 'ascii').upper() + + if qmethod not in _qmethod_map: + raise ValueError( + "qmethod must be one of %s" % list(_qmethod_map.keys()) + ) + + return _qmethod_map[qmethod] + + +def get_dither_seed(dither_seed): + """ + Convert a seed value or indicator to the approprate integer value for + cfitsio + + Parameters + ---------- + dither_seed: number or string + Seed for the subtractive dither. Seeding makes the lossy compression + reproducible. Allowed values are + None or 0 or 'clock': + Return 0, do not set the seed explicitly, use the system clock + negative or 'checksum': + Return -1, means Set the seed based on the data checksum + 1-10_000: + use the input seed + """ + if dither_seed == NOT_SET: + return None + + if isinstance(dither_seed, bytes): + dither_seed = str(dither_seed, 'utf-8') + + if isinstance(dither_seed, str): + dlow = dither_seed.lower() + if dlow == 'clock': + seed_out = 0 + elif dlow == 'checksum': + seed_out = -1 + else: + raise ValueError(f'Bad dither_seed {dither_seed}') + elif dither_seed is None: + seed_out = 0 + else: + # must fit in an int + seed_out = numpy.int32(dither_seed) + + if seed_out > 10_000: + raise ValueError( + f'Got dither_seed {seed_out}, expected avalue <= 10_000' + ) + + return seed_out + + +def check_comptype_img(comptype, dtype_str): + if comptype == NOCOMPRESS: + return + + # if dtype_str == 'i8': + # no i8 allowed for tile-compressed images + # raise ValueError("8-byte integers not supported when " + # "using tile compression") + + if comptype == PLIO_1: + # no unsigned u4/u8 for plio + if dtype_str == 'u4' or dtype_str == 'u8': + raise ValueError( + "Unsigned 4/8-byte integers currently not " + "allowed when writing using PLIO " + "tile compression" + ) + + +def _extract_table_type(type): + """ + Get the numerical table type + """ + if isinstance(type, str): + type = type.lower() + if type[0:7] == 'binary': + table_type = BINARY_TBL + elif type[0:6] == 'ascii': + table_type = ASCII_TBL + else: + raise ValueError( + "table type string should begin with 'binary' or 'ascii' " + "(case insensitive)" + ) + else: + type = int(type) + if type not in [BINARY_TBL, ASCII_TBL]: + raise ValueError( + "table type num should be BINARY_TBL (%d) or " + "ASCII_TBL (%d)" % (BINARY_TBL, ASCII_TBL) + ) + table_type = type + + return table_type + + +# The _compress_map and _qmethod_map only handle +# python-level translations of things like None to their +# C-level meanings. At the C level, we use None to indicate +# not set, as opposed to the values here. +_compress_map = { + None: NOCOMPRESS, + 'RICE': RICE_1, + 'RICE_1': RICE_1, + 'GZIP': GZIP_1, + 'GZIP_1': GZIP_1, + 'GZIP_2': GZIP_2, + 'PLIO': PLIO_1, + 'PLIO_1': PLIO_1, + 'HCOMPRESS': HCOMPRESS_1, + 'HCOMPRESS_1': HCOMPRESS_1, + # In get_compress_type(), we convert the key to str before searching - + # so the keys here must be strings also! + str(NOCOMPRESS): NOCOMPRESS, + str(RICE_1): RICE_1, + str(GZIP_1): GZIP_1, + str(GZIP_2): GZIP_2, + str(PLIO_1): PLIO_1, + str(HCOMPRESS_1): HCOMPRESS_1, +} + +_qmethod_map = { + None: NO_DITHER, + 'NO_DITHER': NO_DITHER, + 'SUBTRACTIVE_DITHER_1': SUBTRACTIVE_DITHER_1, + 'SUBTRACTIVE_DITHER_2': SUBTRACTIVE_DITHER_2, + NO_DITHER: NO_DITHER, + SUBTRACTIVE_DITHER_1: SUBTRACTIVE_DITHER_1, + SUBTRACTIVE_DITHER_2: SUBTRACTIVE_DITHER_2, +} + +_modeprint_map = { + 'r': 'READONLY', + 'rw': 'READWRITE', + 0: 'READONLY', + 1: 'READWRITE', +} +_char_modemap = {'r': 'r', 'rw': 'rw', READONLY: 'r', READWRITE: 'rw'} +_int_modemap = { + 'r': READONLY, + 'rw': READWRITE, + READONLY: READONLY, + READWRITE: READWRITE, +} diff --git a/fitsio/hdu/__init__.py b/fitsio/hdu/__init__.py new file mode 100644 index 0000000..0c6e3ed --- /dev/null +++ b/fitsio/hdu/__init__.py @@ -0,0 +1,14 @@ +from .base import ( # noqa + ANY_HDU, + BINARY_TBL, + ASCII_TBL, + IMAGE_HDU, + _hdu_type_map, +) +from .image import ImageHDU # noqa +from .table import ( # noqa + TableHDU, + AsciiTableHDU, + _table_npy2fits_form, + _npy2fits, +) diff --git a/fitsio/hdu/base.py b/fitsio/hdu/base.py new file mode 100644 index 0000000..f4086a4 --- /dev/null +++ b/fitsio/hdu/base.py @@ -0,0 +1,442 @@ +import re +import copy +import warnings + +from ..util import _stypes, _itypes, _ftypes, FITSRuntimeWarning +from ..header import FITSHDR + +INVALID_HDR_CHARS_RE = re.compile(r"(\?|\*|#)+") +INVALID_HDR_CHARS = {"?", "*", "#"} +ANY_HDU = -1 +IMAGE_HDU = 0 +ASCII_TBL = 1 +BINARY_TBL = 2 + +_hdu_type_map = { + IMAGE_HDU: 'IMAGE_HDU', + ASCII_TBL: 'ASCII_TBL', + BINARY_TBL: 'BINARY_TBL', + 'IMAGE_HDU': IMAGE_HDU, + 'ASCII_TBL': ASCII_TBL, + 'BINARY_TBL': BINARY_TBL, +} + + +class HDUBase(object): + """ + A representation of a FITS HDU + + parameters + ---------- + fits: FITS object + An instance of a _fistio_wrap.FITS object. This is the low-level + python object, not the FITS object defined above. + ext: integer + The extension number. + """ + + def __init__(self, fits, ext, **keys): + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + self._FITS = fits + self._ext = ext + self._ignore_scaling = False + + # init info cache to none + self._cached_info = None + self._filename = self._FITS.filename() + + @property + def _info(self): + if self._cached_info is None: + self._update_info() + return self._cached_info + + def _update_info(self): + """ + Update metadata for this HDU + """ + try: + self._FITS.movabs_hdu(self._ext + 1) + except IOError: + raise RuntimeError("no such hdu") + + self._cached_info = self._FITS.get_hdu_info( + self._ext + 1, self._ignore_scaling + ) + + @property + def ignore_scaling(self): + """ + :return: Flag to indicate whether scaling (BZERO/BSCALE) values should + be ignored. + """ + return self._ignore_scaling + + @ignore_scaling.setter + def ignore_scaling(self, ignore_scaling_flag): + """ + Set the flag to ignore scaling. + """ + old_val = self._ignore_scaling + self._ignore_scaling = ignore_scaling_flag + + # if needed invalidate the info cache so it gets + # updated the next time we access it + if old_val != self._ignore_scaling: + self._cached_info = None + + def get_extnum(self): + """ + Get the extension number + """ + return self._ext + + def get_extname(self): + """ + Get the name for this extension, can be an empty string + """ + name = self._info['extname'] + if name.strip() == '': + name = self._info['hduname'] + return name.strip() + + def get_extver(self): + """ + Get the version for this extension. + + Used when a name is given to multiple extensions + """ + ver = self._info['extver'] + if ver == 0: + ver = self._info['hduver'] + return ver + + def get_exttype(self, num=False): + """ + Get the extension type + + By default the result is a string that mirrors + the enumerated type names in cfitsio + 'IMAGE_HDU', 'ASCII_TBL', 'BINARY_TBL' + which have numeric values + 0 1 2 + send num=True to get the numbers. The values + fitsio.IMAGE_HDU .ASCII_TBL, and .BINARY_TBL + are available for comparison + + parameters + ---------- + num: bool, optional + Return the numeric values. + """ + if num: + return self._info['hdutype'] + else: + name = _hdu_type_map[self._info['hdutype']] + return name + + def get_offsets(self): + """ + returns + ------- + a dictionary with these entries + + header_start: + byte offset from beginning of the file to the start + of the header + data_start: + byte offset from beginning of the file to the start + of the data section + data_end: + byte offset from beginning of the file to the end + of the data section + + Note these are also in the information dictionary, which + you can access with get_info() + """ + return dict( + header_start=self._info['header_start'], + data_start=self._info['data_start'], + data_end=self._info['data_end'], + ) + + def get_info(self): + """ + Get a copy of the internal dictionary holding extension information + """ + return copy.deepcopy(self._info) + + def get_filename(self): + """ + Get a copy of the filename for this fits file + """ + return copy.copy(self._filename) + + def write_checksum(self): + """ + Write the checksum into the header for this HDU. + + Computes the checksum for the HDU, both the data portion alone (DATASUM + keyword) and the checksum complement for the entire HDU (CHECKSUM). + + returns + ------- + A dict with keys 'datasum' and 'hdusum' + """ + ret = self._FITS.write_checksum(self._ext + 1) + self._cached_info = None # invalidate info cache + return ret + + def verify_checksum(self): + """ + Verify the checksum in the header for this HDU. + """ + res = self._FITS.verify_checksum(self._ext + 1) + if res['dataok'] != 1: + raise ValueError("data checksum failed") + if res['hduok'] != 1: + raise ValueError("hdu checksum failed") + + def write_comment(self, comment): + """ + Write a comment into the header + """ + self._FITS.write_comment(self._ext + 1, str(comment)) + self._cached_info = None # invalidate info cache + + def write_history(self, history): + """ + Write history text into the header + """ + self._FITS.write_history(self._ext + 1, str(history)) + self._cached_info = None # invalidate info cache + + def _write_continue(self, value): + """ + Write history text into the header + """ + self._FITS.write_continue(self._ext + 1, str(value)) + self._cached_info = None # invalidate info cache + + def write_key(self, name, value, comment=""): + """ + Write the input value to the header + + parameters + ---------- + name: string + Name of keyword to write/update + value: scalar + Value to write, can be string float or integer type, + including numpy scalar types. + comment: string, optional + An optional comment to write for this key + + Notes + ----- + Write COMMENT and HISTORY using the write_comment and write_history + methods + """ + + if name is None: + # we write a blank keyword and the rest is a comment + # string + + if not isinstance(comment, _stypes): + raise ValueError( + 'when writing blank key the value must be a string' + ) + + # this might be longer than 80 but that's ok, the routine + # will take care of it + # card = ' ' + str(comment) + card = ' ' + str(comment) + self._FITS.write_record( + self._ext + 1, + card, + ) + + elif value is None: + self._FITS.write_undefined_key( + self._ext + 1, str(name), str(comment) + ) + + elif isinstance(value, bool): + if value: + v = 1 + else: + v = 0 + self._FITS.write_logical_key( + self._ext + 1, str(name), v, str(comment) + ) + elif isinstance(value, _stypes): + self._FITS.write_string_key( + self._ext + 1, str(name), str(value), str(comment) + ) + elif isinstance(value, _ftypes): + self._FITS.write_double_key( + self._ext + 1, str(name), float(value), str(comment) + ) + elif isinstance(value, _itypes): + self._FITS.write_long_long_key( + self._ext + 1, str(name), int(value), str(comment) + ) + elif isinstance(value, (tuple, list)): + vl = [str(el) for el in value] + sval = ','.join(vl) + self._FITS.write_string_key( + self._ext + 1, str(name), sval, str(comment) + ) + else: + sval = str(value) + mess = ( + "warning, keyword '%s' has non-standard " + "value type %s, " + "Converting to string: '%s'" + ) + warnings.warn(mess % (name, type(value), sval), FITSRuntimeWarning) + self._FITS.write_string_key( + self._ext + 1, str(name), sval, str(comment) + ) + + self._cached_info = None # invalidate info cache + + def write_keys(self, records_in, clean=True): + """ + Write the keywords to the header. + + parameters + ---------- + records: FITSHDR or list or dict + Can be one of these: + - FITSHDR object + - list of dictionaries containing 'name','value' and optionally + a 'comment' field; the order is preserved. + - a dictionary of keyword-value pairs; no comments are written + in this case, and the order is arbitrary. + clean: boolean + If True, trim out the standard fits header keywords that are + created on HDU creation (e.g., EXTEND, SIMPLE, STTYPE, TFORM, + TDIM, XTENSION, BITPIX, NAXIS, etc.) from the input records + before they are written to the current fits file. + + Notes + ----- + Input keys named COMMENT and HISTORY are written using the + write_comment and write_history methods. + """ + + if isinstance(records_in, FITSHDR): + hdr = records_in + else: + hdr = FITSHDR(records_in) + + if clean: + is_table = hasattr(self, '_table_type_str') + # is_table = isinstance(self, TableHDU) + hdr.clean(is_table=is_table) + + for r in hdr.records(): + name = r['name'] + if name is not None: + name = name.upper() + + if INVALID_HDR_CHARS_RE.search(name): + raise RuntimeError( + "header key '%s' has invalid characters! " + "Characters in %s are not allowed!" + % (name, INVALID_HDR_CHARS) + ) + + value = r['value'] + + if name == 'COMMENT': + self.write_comment(value) + elif name == 'HISTORY': + self.write_history(value) + elif name == 'CONTINUE': + self._write_continue(value) + else: + comment = r.get('comment', '') + self.write_key(name, value, comment=comment) + + self._cached_info = None # invalidate info cache + + def read_header(self): + """ + Read the header as a FITSHDR + + The FITSHDR allows access to the values and comments by name and + number. + """ + # note converting strings + return FITSHDR(self.read_header_list()) + + def read_header_list(self): + """ + Read the header as a list of dictionaries. + + You will usually use read_header instead, which just sends the output + of this functioin to the constructor of a FITSHDR, which allows access + to the values and comments by name and number. + + Each dictionary is + 'name': the keyword name + 'value': the value field as a string + 'comment': the comment field as a string. + """ + return self._FITS.read_header(self._ext + 1) + + def delete_key(self, name): + """ + Delete the key from the header. + + parameters + ---------- + name: string + Name of keyword to delete. + """ + self.delete_keys([name]) + + def delete_keys(self, names): + """ + Delete the keys from the header. + + parameters + ---------- + names: iterable of keys + Names of keywords to delete. + """ + for name in names: + self._FITS.delete_key(self._ext + 1, str(name)) + self._cached_info = None # invalidate info cache + + def _get_repr_list(self): + """ + Get some representation data common to all HDU types + """ + spacing = ' ' * 2 + text = [''] + text.append("%sfile: %s" % (spacing, self._filename)) + text.append("%sextension: %d" % (spacing, self._info['hdunum'] - 1)) + text.append( + "%stype: %s" % (spacing, _hdu_type_map[self._info['hdutype']]) + ) + + extname = self.get_extname() + if extname != "": + text.append("%sextname: %s" % (spacing, extname)) + extver = self.get_extver() + if extver != 0: + text.append("%sextver: %s" % (spacing, extver)) + + return text, spacing diff --git a/fitsio/hdu/image.py b/fitsio/hdu/image.py new file mode 100644 index 0000000..63db0f4 --- /dev/null +++ b/fitsio/hdu/image.py @@ -0,0 +1,550 @@ +""" +image HDU classes for fitslib, part of the fitsio package. + +See the main docs at https://github.com/esheldon/fitsio + + Copyright (C) 2011 Erin Sheldon, BNL. erin dot sheldon at gmail dot com + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" + +from __future__ import with_statement, print_function + +import numpy + +from math import floor +from .base import HDUBase, IMAGE_HDU +from ..util import ( + IS_PY3, + array_to_native, + copy_if_needed, + _nonfinite_as_cfitsio_floating_null_value, +) + +# for python3 compat +if IS_PY3: + xrange = range + + +class ImageHDU(HDUBase): + def _update_info(self): + """ + Call parent method and make sure this is in fact a + image HDU. Set dims in C order + """ + super(ImageHDU, self)._update_info() + + if self._info['hdutype'] != IMAGE_HDU: + mess = "Extension %s is not a Image HDU" % self.ext + raise ValueError(mess) + + # convert to c order + if 'dims' in self._info: + self._info['dims'] = list(reversed(self._info['dims'])) + + def has_data(self): + """ + Determine if this HDU has any data + + For images, check that the dimensions are not zero. + + For tables, check that the row count is not zero + """ + ndims = self._info.get('ndims', 0) + if ndims == 0: + return False + else: + return True + + def is_compressed(self): + """ + returns true of this extension is compressed + """ + return self._info['is_compressed_image'] == 1 + + def get_comptype(self): + """ + Get the compression type. + + None if the image is not compressed. + """ + return self._info['comptype'] + + def get_dims(self): + """ + get the shape of the image. Returns () for empty + """ + if self._info['ndims'] != 0: + dims = self._info['dims'] + else: + dims = () + + return dims + + def reshape(self, dims): + """ + reshape an existing image to the requested dimensions + + If the new shape is bigger than the current shape, + the existing values in the image are "wrapped" around in C + order, via the process of + + 1. flattening the image in C order + 2. appending zeros to the image so that it matches the new + total size + 3. reshaping the image to the new dimensions + + If the new shape is smaller than the current image, the current + image is flattened, trunctaed to the new total length, and then + reshaped to the new shape. + + parameters + ---------- + dims: sequence + Any sequence convertible to i8 + """ + + adims = numpy.array(dims, ndmin=1, dtype='i8') + # we have to reverse the dimensions here since cfitsio + # uses fortran order + self._FITS.reshape_image(self._ext + 1, adims[::-1]) + self._cached_info = None # invalidate info cache + + def write(self, img, start=0, **keys): + """ + Write the image into this HDU + + If data already exist in this HDU, they will be overwritten. If the + image to write is larger than the image on disk, or if the start + position is such that the write would extend beyond the existing + dimensions, the on-disk image is expanded. + + parameters + ---------- + img: ndarray + A simple numpy ndarray + start: integer or sequence + Where to start writing data. Can be an integer offset + into the entire array, or a sequence determining where + in N-dimensional space to start. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if img.dtype.fields is not None: + raise ValueError("got recarray, expected regular ndarray") + if img.size == 0: + raise ValueError("data must have at least 1 row") + + # data must be c-contiguous and native byte order + if not img.flags['C_CONTIGUOUS']: + # this always makes a copy + img_send = numpy.ascontiguousarray(img) + img_send = array_to_native(img_send, inplace=True) + else: + img_send = array_to_native(img, inplace=False) + + if IS_PY3 and img_send.dtype.char == 'U': + # for python3, we convert unicode to ascii + # this will error if the character is not in ascii + img_send = img_send.astype('S', copy=copy_if_needed) + + # see if we need to resize the image + if self.has_data(): + self._expand_if_needed(self.get_dims(), img.shape, start) + dims = self.get_dims() + + if numpy.isscalar(start): + start = numpy.unravel_index(start, dims) + + if all(od == nd for od, nd in zip(dims, img.shape)) and all( + st == 0 for st in start + ): + # we are replacing the whole image, so no need to + # write a subset + write_subset = False + else: + write_subset = True + else: + write_subset = False + + with _nonfinite_as_cfitsio_floating_null_value( + img_send, self.is_compressed() + ) as img_send_any_nan: + img_send, any_nan = img_send_any_nan + if not write_subset: + # write in image at start in a single pass + offset = 0 + self._FITS.write_image( + self._ext + 1, + img_send, + offset + 1, + 1 if any_nan else 0, + ) + else: + if not any_nan and not self.is_compressed(): + firstpixel = numpy.array(start, ndmin=1, dtype='i8') + # lastpixel is the index of the lastpixel so subtract 1 + lastpixel = ( + firstpixel + + numpy.array(img_send.shape, ndmin=1, dtype='i8') + - 1 + ) + + # we have to reverse the dimensions here since cfitsio + # uses fortran order and offset by 1 for fortan indexing + firstpixel = firstpixel[::-1] + 1 + lastpixel = lastpixel[::-1] + 1 + + self._FITS.write_subset( + self._ext + 1, img_send, firstpixel, lastpixel + ) + else: + # the C API doesn't support nan handling w/ rectangular + # subsets, so emulate in python + # go "row by row" but in more than two dimensions + ndims = len(dims) + for index in numpy.ndindex(*(img_send.shape[:-1])): + new_start = [ + start[i] + index[i] for i in range(ndims - 1) + ] + new_start += [start[-1]] + offset = _convert_full_start_to_offset(dims, new_start) + img_slice = tuple( + [slice(ns, ns + 1) for ns in index] + ) + (slice(None),) + self._FITS.write_image( + self._ext + 1, + img_send[img_slice], + offset + 1, + 1 if any_nan else 0, + ) + + self._cached_info = None # invalidate info cache + + def read(self, **keys): + """ + Read the image. + + If the HDU is an IMAGE_HDU, read the corresponding image. Compression + and scaling are dealt with properly. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if not self.has_data(): + return None + + dtype, shape = self._get_dtype_and_shape() + array = numpy.zeros(shape, dtype=dtype) + self._FITS.read_image(self._ext + 1, array) + return array + + def _get_dtype_and_shape(self): + """ + Get the numpy dtype and shape for image + """ + npy_dtype = self._get_image_numpy_dtype() + + if self._info['ndims'] != 0: + shape = self._info['dims'] + else: + raise IOError("no image present in HDU") + + return npy_dtype, shape + + def _get_image_numpy_dtype(self): + """ + Get the numpy dtype for the image + """ + try: + ftype = self._info['img_equiv_type'] + npy_type = _image_bitpix2npy[ftype] + except KeyError: + raise KeyError("unsupported fits data type: %d" % ftype) + + return npy_type + + def __getitem__(self, arg): + """ + Get data from an image using python [] slice notation. + + e.g., [2:25, 4:45]. + """ + return self._read_image_slice(arg) + + def _read_image_slice(self, arg): + """ + workhorse to read a slice + """ + if 'ndims' not in self._info: + raise ValueError("Attempt to slice empty extension") + + if isinstance(arg, slice): + # one-dimensional, e.g. 2:20 + return self._read_image_slice((arg,)) + + if not isinstance(arg, tuple): + raise ValueError( + "arguments must be slices, one for each " + "dimension, e.g. [2:5] or [2:5,8:25] etc." + ) + + # should be a tuple of slices, one for each dimension + # e.g. [2:3, 8:100] + nd = len(arg) + if nd != self._info['ndims']: + raise ValueError( + "Got slice dimensions %d, " + "expected %d" % (nd, self._info['ndims']) + ) + + targ = arg + arg = [] + for a in targ: + if isinstance(a, slice): + arg.append(a) + elif isinstance(a, int): + arg.append(slice(a, a + 1, 1)) + else: + raise ValueError("arguments must be slices, e.g. 2:12") + + dims = self._info['dims'] + arrdims = [] + first = [] + last = [] + steps = [] + npy_dtype = self._get_image_numpy_dtype() + + # check the args and reverse dimensions since + # fits is backwards from numpy + dim = 0 + for slc in arg: + start = slc.start + stop = slc.stop + step = slc.step + + if start is None: + start = 0 + if stop is None: + stop = dims[dim] + if step is None: + # Ensure sane defaults. + if start <= stop: + step = 1 + else: + step = -1 + + # Sanity checks for proper syntax. + if ( + (step > 0 and stop < start) + or (step < 0 and start < stop) + or (start == stop) + ): + return numpy.empty(0, dtype=npy_dtype) + + if start < 0: + start = dims[dim] + start + if start < 0: + raise IndexError("Index out of bounds") + + if stop < 0: + stop = dims[dim] + start + 1 + + # move to 1-offset + start = start + 1 + + if stop > dims[dim]: + stop = dims[dim] + + if stop < start: + # A little black magic here. The stop is offset by 2 to + # accommodate the 1-offset of CFITSIO, and to move past the end + # pixel to get the complete set after it is flipped along the + # axis. Maybe there is a clearer way to accomplish what this + # offset is glossing over. + # @at88mph 2019.10.10 + stop = stop + 2 + + first.append(start) + last.append(stop) + + # Negative step values are not used in CFITSIO as the dimension is + # already properly calcualted. + # @at88mph 2019.10.21 + steps.append(abs(step)) + arrdims.append(int(floor((stop - start) / step)) + 1) + + dim += 1 + + first.reverse() + last.reverse() + steps.reverse() + first = numpy.array(first, dtype='i8') + last = numpy.array(last, dtype='i8') + steps = numpy.array(steps, dtype='i8') + + array = numpy.zeros(arrdims, dtype=npy_dtype) + self._FITS.read_image_slice( + self._ext + 1, first, last, steps, self._ignore_scaling, array + ) + return array + + def _expand_if_needed(self, dims, write_dims, start): + """ + expand the on-disk image if the indended write will extend + beyond the existing dimensions + """ + ndim = len(dims) + idim = len(write_dims) + + if idim != ndim: + raise ValueError( + "When expanding " + "an existing image while writing, the input image " + "must have the same number of dimensions " + "as the original. " + "Got %d instead of %d" % (idim, ndim) + ) + + if numpy.isscalar(start): + if len(dims) > 1: + try: + _start = numpy.unravel_index(start, dims) + except Exception: + # the unravel_index call fails when start is beyond + # end of the existing array. + # this means we are expanding the image and so we should + # error + raise ValueError( + "When expanding " + "an existing image while writing, the start keyword " + "must have the same number of dimensions " + "as the image or be exactly 0, got %s " % start + ) + else: + _start = [start] + else: + _start = start + + new_dims = [] + for i in xrange(ndim): + required_dim = _start[i] + write_dims[i] + + if required_dim < dims[i]: + # careful not to shrink the image! + dimsize = dims[i] + else: + dimsize = required_dim + + new_dims.append(dimsize) + + if any(nd != od for nd, od in zip(new_dims, dims)): + if numpy.isscalar(start) and len(dims) > 1: + if start != 0: + raise ValueError( + "When expanding " + "an existing image while writing, the start keyword " + "must have the same number of dimensions " + "as the image or be exactly 0, got %s " % start + ) + self.reshape(new_dims) + + def __repr__(self): + """ + Representation for ImageHDU + """ + text, spacing = self._get_repr_list() + text.append("%simage info:" % spacing) + cspacing = ' ' * 4 + + # need this check for when we haven't written data yet + if 'ndims' in self._info: + if self._info['comptype'] is not None: + text.append( + "%scompression: %s" % (cspacing, self._info['comptype']) + ) + + if self._info['ndims'] != 0: + dimstr = [str(d) for d in self._info['dims']] + dimstr = ",".join(dimstr) + else: + dimstr = '' + + dt = _image_bitpix2npy[self._info['img_equiv_type']] + text.append("%sdata type: %s" % (cspacing, dt)) + text.append("%sdims: [%s]" % (cspacing, dimstr)) + + text = '\n'.join(text) + return text + + +def _convert_full_start_to_offset(dims, start): + # convert to scalar offset + # note we use the on-disk data type to get itemsize + ndim = len(dims) + + # convert sequence to pixel start + if len(start) != ndim: + m = "start has len %d, which does not match requested dims %d" + raise ValueError(m % (len(start), ndim)) + + # MRB: I changed this to use the numpy util below. + # I have left the old code here for posterity. + # I checked that they give the same answer. + # # this is really strides / itemsize + # strides = [1] + # for i in xrange(1, ndim): + # strides.append(strides[i - 1] * dims[ndim - i]) + + # strides.reverse() + # s = start + # start_index = sum([s[i] * strides[i] for i in xrange(ndim)]) + + # return start_index + + return numpy.ravel_multi_index(start, dims) + + +# remember, you should be using the equivalent image type for this +_image_bitpix2npy = { + 8: 'u1', + 10: 'i1', + 16: 'i2', + 20: 'u2', + 32: 'i4', + 40: 'u4', + 64: 'i8', + 80: 'u8', + -32: 'f4', + -64: 'f8', +} diff --git a/fitsio/hdu/table.py b/fitsio/hdu/table.py new file mode 100644 index 0000000..7ccd887 --- /dev/null +++ b/fitsio/hdu/table.py @@ -0,0 +1,2845 @@ +""" +image HDU classes for fitslib, part of the fitsio package. + +See the main docs at https://github.com/esheldon/fitsio + + Copyright (C) 2011 Erin Sheldon, BNL. erin dot sheldon at gmail dot com + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" + +from __future__ import with_statement, print_function +import copy +import warnings +from functools import reduce + +import numpy as np + +from ..util import ( + IS_PY3, + isstring, + isinteger, + is_object, + fields_are_object, + array_to_native, + array_to_native_c, + FITSRuntimeWarning, + mks, + copy_if_needed, +) +from .base import HDUBase, ASCII_TBL, IMAGE_HDU, _hdu_type_map + +# for python3 compat +if IS_PY3: + xrange = range + + +class TableHDU(HDUBase): + """ + A table HDU + + parameters + ---------- + fits: FITS object + An instance of a _fistio_wrap.FITS object. This is the low-level + python object, not the FITS object defined above. + ext: integer + The extension number. + lower: bool, optional + If True, force all columns names to lower case in output + upper: bool, optional + If True, force all columns names to upper case in output + trim_strings: bool, optional + If True, trim trailing spaces from strings. Default is False. + vstorage: string, optional + Set the default method to store variable length columns. Can be + 'fixed' or 'object'. See docs on fitsio.FITS for details. + case_sensitive: bool, optional + Match column names and extension names with case-sensitivity. Default + is False. + iter_row_buffer: integer + Number of rows to buffer when iterating over table HDUs. + Default is 1. + write_bitcols: bool, optional + If True, write logicals a a bit column. Default is False. + """ + + def __init__( + self, + fits, + ext, + lower=False, + upper=False, + trim_strings=False, + vstorage='fixed', + case_sensitive=False, + iter_row_buffer=1, + write_bitcols=False, + **keys, + ): + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + # NOTE: The defaults of False above cannot be changed since they + # are or'ed with the method defaults below. + super(TableHDU, self).__init__(fits, ext) + + self.lower = lower + self.upper = upper + self.trim_strings = trim_strings + + self._vstorage = vstorage + self.case_sensitive = case_sensitive + self._iter_row_buffer = iter_row_buffer + self.write_bitcols = write_bitcols + + if self._info['hdutype'] == ASCII_TBL: + self._table_type_str = 'ascii' + else: + self._table_type_str = 'binary' + + def get_nrows(self): + """ + Get number of rows in the table. + """ + nrows = self._info.get('nrows', None) + if nrows is None: + raise ValueError("nrows not in info table; this is a bug") + return nrows + + def get_colnames(self): + """ + Get a copy of the column names for a table HDU + """ + return copy.copy(self._info["colnames"]) + + def get_colname(self, colnum): + """ + Get the name associated with the given column number + + parameters + ---------- + colnum: integer + The number for the column, zero offset + """ + if colnum < 0 or colnum > (self._info["ncol"] - 1): + raise ValueError( + "colnum out of range [0,%s-1]" % self._info["ncol"] + ) + return self._info["colnames"][colnum] + + def get_vstorage(self): + """ + Get a string representing the storage method for variable length + columns + """ + return copy.copy(self._vstorage) + + def has_data(self): + """ + Determine if this HDU has any data + + Check that the row count is not zero + """ + if self._info['nrows'] > 0: + return True + else: + return False + + def where(self, expression, firstrow=None, lastrow=None): + """ + Return the indices where the expression evaluates to true. + + parameters + ---------- + expression: string + A fits row selection expression. E.g. + "x > 3 && y < 5" + firstrow, lastrow : int + Range of rows for evaluation. This follows the Python list + slice convention that the last element is not included. + """ + if firstrow is None: + firstrow = 0 + elif firstrow < 0: + raise ValueError('firstrow cannot be negative') + if lastrow is None: + lastrow = self._info['nrows'] + elif lastrow < firstrow: + raise ValueError('lastrow cannot be less than firstrow') + elif lastrow > self._info['nrows']: + raise ValueError('lastrow cannot be greater than nrows') + nrows = lastrow - firstrow + return self._FITS.where(self._ext + 1, expression, firstrow + 1, nrows) + + def write( + self, data, firstrow=0, columns=None, names=None, slow=False, **keys + ): + """ + Write data into this HDU + + parameters + ---------- + data: ndarray or list of ndarray + A numerical python array. Should be an ordinary array for image + HDUs, should have fields for tables. To write an ordinary array to + a column in a table HDU, use write_column. If data already exists + in this HDU, it will be overwritten. See the append(() method to + append new rows to a table HDU. + firstrow: integer, optional + At which row you should begin writing to tables. Be sure you know + what you are doing! For appending see the append() method. + Default 0. + columns: list, optional + If data is a list of arrays, you must send columns as a list + of names or column numbers. You can also use the `names` keyword + argument. + names: list, optional + If data is a list of arrays, you must send columns as a list + of names or column numbers. You can also use the `columns` keyword + argument. + slow: bool, optional + If True, use a slower method to write one column at a time. Useful + for debugging. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + isrec = False + if isinstance(data, (list, dict)): + if isinstance(data, list): + data_list = data + if columns is not None: + columns_all = columns + elif names is not None: + columns_all = names + else: + raise ValueError( + "you must send `columns` or `names` " + "with a list of arrays" + ) + else: + columns_all = list(data.keys()) + data_list = [data[n] for n in columns_all] + + colnums_all = [self._extract_colnum(c) for c in columns_all] + names = [self.get_colname(c) for c in colnums_all] + + isobj = np.zeros(len(data_list), dtype=bool) + for i in xrange(len(data_list)): + isobj[i] = is_object(data_list[i]) + + else: + if data.dtype.fields is None: + raise ValueError( + "You are writing to a table, so I expected " + "an array with fields as input. If you want " + "to write a simple array, you should use " + "write_column to write to a single column, " + "or instead write to an image hdu" + ) + + if data.shape == (): + raise ValueError("cannot write data with shape ()") + + isrec = True + names = data.dtype.names + # only write object types (variable-length columns) after + # writing the main table + isobj = fields_are_object(data) + + data_list = [] + colnums_all = [] + for i, name in enumerate(names): + colnum = self._extract_colnum(name) + data_list.append(data[name]) + colnums_all.append(colnum) + + if slow: + for i, name in enumerate(names): + if not isobj[i]: + self.write_column(name, data_list[i], firstrow=firstrow) + else: + nonobj_colnums = [] + nonobj_arrays = [] + for i in xrange(len(data_list)): + if not isobj[i]: + nonobj_colnums.append(colnums_all[i]) + if isrec: + # this still leaves possibility of f-order sub-arrays.. + colref = array_to_native(data_list[i], inplace=False) + else: + colref = array_to_native_c(data_list[i], inplace=False) + + if IS_PY3 and colref.dtype.char == 'U': + # for python3, we convert unicode to ascii + # this will error if the character is not in ascii + colref = colref.astype('S', copy=copy_if_needed) + + nonobj_arrays.append(colref) + + for tcolnum, tdata in zip(nonobj_colnums, nonobj_arrays): + self._verify_column_data(tcolnum, tdata) + + if len(nonobj_arrays) > 0: + self._FITS.write_columns( + self._ext + 1, + nonobj_colnums, + nonobj_arrays, + firstrow=firstrow + 1, + write_bitcols=self.write_bitcols, + ) + + # writing the object arrays always occurs the same way + # need to make sure this works for array fields + for i, name in enumerate(names): + if isobj[i]: + self.write_var_column(name, data_list[i], firstrow=firstrow) + + self._cached_info = None # invalidate info cache + + def write_column(self, column, data, firstrow=0, **keys): + """ + Write data to a column in this HDU + + This HDU must be a table HDU. + + parameters + ---------- + column: scalar string/integer + The column in which to write. Can be the name or number (0 offset) + data: ndarray + Numerical python array to write. This should match the + shape of the column. You are probably better using + fits.write_table() to be sure. + firstrow: integer, optional + At which row you should begin writing. Be sure you know what you + are doing! For appending see the append() method. Default 0. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + colnum = self._extract_colnum(column) + + # need it to be contiguous and native byte order. For now, make a + # copy. but we may be able to avoid this with some care. + + if not data.flags['C_CONTIGUOUS']: + # this always makes a copy + data_send = np.ascontiguousarray(data) + # this is a copy, we can make sure it is native + # and modify in place if needed + data_send = array_to_native(data_send, inplace=True) + else: + # we can avoid the copy with a try-finally block and + # some logic + data_send = array_to_native(data, inplace=False) + + if IS_PY3 and data_send.dtype.char == 'U': + # for python3, we convert unicode to ascii + # this will error if the character is not in ascii + data_send = data_send.astype('S', copy=copy_if_needed) + + self._verify_column_data(colnum, data_send) + + self._FITS.write_columns( + self._ext + 1, + [colnum], + [data_send], + firstrow=firstrow + 1, + write_bitcols=self.write_bitcols, + ) + + del data_send + self._cached_info = None # invalidate info cache + + def _verify_column_data(self, colnum, data): + """ + verify the input data is of the correct type and shape + """ + this_dt = data.dtype.descr[0] + + if len(data.shape) > 2: + this_shape = data.shape[1:] + elif len(data.shape) == 2 and data.shape[1] > 1: + this_shape = data.shape[1:] + else: + this_shape = () + + this_npy_type = this_dt[1][1:] + + npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) + info = self._info['colinfo'][colnum] + + if npy_type[0] in ['>', '<', '|']: + npy_type = npy_type[1:] + + col_name = info['name'] + col_tdim = info['tdim'] + col_shape = _tdim2shape( + col_tdim, col_name, is_string=(npy_type[0] == 'S') + ) + + if col_shape is None: + if this_shape == (): + this_shape = None + + if col_shape is not None and not isinstance(col_shape, tuple): + col_shape = (col_shape,) + + # this mismatch is OK + if npy_type == 'i1' and this_npy_type == 'b1': + this_npy_type = 'i1' + + if isinstance(self, AsciiTableHDU): + # we don't enforce types exact for ascii + if npy_type == 'i8' and this_npy_type in ['i2', 'i4']: + this_npy_type = 'i8' + elif npy_type == 'f8' and this_npy_type == 'f4': + this_npy_type = 'f8' + + if this_npy_type != npy_type: + raise ValueError( + "bad input data for column '%s': " + "expected '%s', got '%s'" % (col_name, npy_type, this_npy_type) + ) + + if this_shape != col_shape: + raise ValueError( + "bad input shape for column '%s': " + "expected '%s', got '%s'" % (col_name, col_shape, this_shape) + ) + + def write_var_column(self, column, data, firstrow=0, **keys): + """ + Write data to a variable-length column in this HDU + + This HDU must be a table HDU. + + parameters + ---------- + column: scalar string/integer + The column in which to write. Can be the name or number (0 offset) + column: ndarray + Numerical python array to write. This must be an object array. + firstrow: integer, optional + At which row you should begin writing. Be sure you know what you + are doing! For appending see the append() method. Default 0. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if not is_object(data): + raise ValueError( + "Only object fields can be written to variable-length arrays" + ) + colnum = self._extract_colnum(column) + + self._FITS.write_var_column( + self._ext + 1, colnum + 1, data, firstrow=firstrow + 1 + ) + self._cached_info = None # invalidate info cache + + def insert_column( + self, name, data, colnum=None, write_bitcols=None, **keys + ): + """ + Insert a new column. + + parameters + ---------- + name: string + The column name + data: + The data to write into the new column. + colnum: int, optional + The column number for the new column, zero-offset. Default + is to add the new column after the existing ones. + write_bitcols: bool, optional + If set, write logical as bit cols. This can over-ride the + internal class setting. Default of None respects the inner + class setting. + + Notes + ----- + This method is used un-modified by ascii tables as well. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if write_bitcols is None: + write_bitcols = self.write_bitcols + + if name in self._info["colnames"]: + raise ValueError("column '%s' already exists" % name) + + if IS_PY3 and data.dtype.char == 'U': + # fast dtype conversion using an empty array + # we could hack at the actual text description, but using + # the numpy API is probably safer + # this also avoids doing a dtype conversion on every array + # element which could b expensive + descr = np.empty(1).astype(data.dtype).astype('S').dtype.descr + else: + descr = data.dtype.descr + + if len(descr) > 1: + raise ValueError( + "you can only insert a single column, requested: %s" % descr + ) + + this_descr = descr[0] + this_descr = [name, this_descr[1]] + if len(data.shape) > 1: + this_descr += [data.shape[1:]] + this_descr = tuple(this_descr) + + name, fmt, dims = _npy2fits( + this_descr, + table_type=self._table_type_str, + write_bitcols=write_bitcols, + ) + if dims is not None: + dims = [dims] + + if colnum is None: + new_colnum = len(self._info['colinfo']) + 1 + else: + new_colnum = colnum + 1 + + self._FITS.insert_col(self._ext + 1, new_colnum, name, fmt, tdim=dims) + + self._cached_info = None # invalidate info cache + + self.write_column(name, data) + + def append(self, data, columns=None, names=None, **keys): + """ + Append new rows to a table HDU + + parameters + ---------- + data: ndarray or list of arrays + A numerical python array with fields (recarray) or a list of + arrays. Should have the same fields as the existing table. If only + a subset of the table columns are present, the other columns are + filled with zeros. + columns: list, optional + If data is a list of arrays, you must send columns as a list + of names or column numbers. You can also use the `names` keyword + argument. + names: list, optional + If data is a list of arrays, you must send columns as a list + of names or column numbers. You can also use the `columns` keyword + argument. + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + firstrow = self._info['nrows'] + self.write(data, firstrow=firstrow, columns=columns, names=names) + + def delete_rows(self, rows): + """ + Delete rows from the table + + parameters + ---------- + rows: sequence or slice + The exact rows to delete as a sequence, or a slice. + + examples + -------- + # delete a range of rows + with fitsio.FITS(fname,'rw') as fits: + fits['mytable'].delete_rows(slice(3,20)) + + # delete specific rows + with fitsio.FITS(fname,'rw') as fits: + rows2delete = [3,88,76] + fits['mytable'].delete_rows(rows2delete) + """ + + if rows is None: + return + + # extract and convert to 1-offset for C routine + if isinstance(rows, slice): + rows = self._process_slice(rows) + if rows.step is not None and rows.step != 1: + rows = np.arange( + rows.start + 1, + rows.stop + 1, + rows.step, + ) + else: + # rows must be 1-offset + rows = slice(rows.start + 1, rows.stop + 1) + else: + rows, sortind = self._extract_rows(rows, sort=True) + # rows must be 1-offset + rows += 1 + + if isinstance(rows, slice): + self._FITS.delete_row_range(self._ext + 1, rows.start, rows.stop) + else: + if rows.size == 0: + return + + self._FITS.delete_rows(self._ext + 1, rows) + + self._cached_info = None # invalidate info cache + + def resize(self, nrows, front=False): + """ + Resize the table to the given size, removing or adding rows as + necessary. Note if expanding the table at the end, it is more + efficient to use the append function than resizing and then + writing. + + New added rows are zerod, except for 'i1', 'u2' and 'u4' data types + which get -128,32768,2147483648 respectively + + parameters + ---------- + nrows: int + new size of table + front: bool, optional + If True, add or remove rows from the front. Default + is False + """ + + nrows_current = self.get_nrows() + if nrows == nrows_current: + return + + if nrows < nrows_current: + rowdiff = nrows_current - nrows + if front: + # delete from the front + start = 0 + stop = rowdiff + else: + # delete from the back + start = nrows + stop = nrows_current + + self.delete_rows(slice(start, stop)) + else: + rowdiff = nrows - nrows_current + if front: + # in this case zero is what we want, since the code inserts + firstrow = 0 + else: + firstrow = nrows_current + self._FITS.insert_rows(self._ext + 1, firstrow, rowdiff) + + self._cached_info = None # invalidate info cache + + def read( + self, + columns=None, + rows=None, + vstorage=None, + upper=False, + lower=False, + trim_strings=False, + **keys, + ): + """ + Read data from this HDU + + By default, all data are read. You can set the `columns` and/or + `rows` keywords to read subsets of the data. + + Table data is read into a numpy recarray. To get a single column as + a numpy.ndarray, use the `read_column` method. + + Slice notation is also supported for `TableHDU` types. + + >>> fits = fitsio.FITS(filename) + >>> fits[ext][:] + >>> fits[ext][2:5] + >>> fits[ext][200:235:2] + >>> fits[ext][rows] + >>> fits[ext][cols][rows] + + parameters + ---------- + columns: optional + An optional set of columns to read from table HDUs. Default is to + read all. Can be string or number. If a sequence, a recarray + is always returned. If a scalar, an ordinary array is returned. + rows: optional + An optional list of rows to read from table HDUS. Default is to + read all. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if columns is not None: + data = self.read_columns( + columns, + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + elif rows is not None: + # combinations of row and column subsets are covered by + # read_columns so we pass colnums=None here to get all columns + data = self.read_rows( + rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + else: + data = self._read_all( + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + + return data + + def _read_all( + self, + vstorage=None, + upper=False, + lower=False, + trim_strings=False, + colnums=None, + **keys, + ): + """ + Read all data in the HDU. + + parameters + ---------- + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + colnums: integer array, optional + The column numbers, 0 offset + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + dtype, offsets, isvar = self.get_rec_dtype( + colnums=colnums, vstorage=vstorage + ) + + (w,) = np.where(isvar == True) # noqa + has_tbit = self._check_tbit() + + if w.size > 0: + if vstorage is None: + _vstorage = self._vstorage + else: + _vstorage = vstorage + colnums = self._extract_colnums() + rows = None + sortind = None + array = self._read_rec_with_var( + colnums, + rows, + sortind, + dtype, + offsets, + isvar, + _vstorage, + ) + elif has_tbit: + # drop down to read_columns since we can't stuff into a + # contiguous array + colnums = self._extract_colnums() + array = self.read_columns( + colnums, + rows=None, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + else: + firstrow = 1 # noqa - not used? + nrows = self._info['nrows'] + array = np.zeros(nrows, dtype=dtype) + + self._FITS.read_as_rec(self._ext + 1, 1, nrows, array) + + array = self._maybe_decode_fits_ascii_strings_to_unicode_py3(array) + + for colnum, name in enumerate(array.dtype.names): + self._rescale_and_convert_field_inplace( + array, + name, + self._info['colinfo'][colnum]['tscale'], + self._info['colinfo'][colnum]['tzero'], + ) + + if self.lower or lower: + _names_to_lower_if_recarray(array) + elif self.upper or upper: + _names_to_upper_if_recarray(array) + + self._maybe_trim_strings(array, trim_strings=trim_strings) + return array + + def read_column( + self, + col, + rows=None, + vstorage=None, + upper=False, + lower=False, + trim_strings=False, + **keys, + ): + """ + Read the specified column + + Alternatively, you can use slice notation + + >>> fits=fitsio.FITS(filename) + >>> fits[ext][colname][:] + >>> fits[ext][colname][2:5] + >>> fits[ext][colname][200:235:2] + >>> fits[ext][colname][rows] + + Note, if reading multiple columns, it is more efficient to use + read(columns=) or slice notation with a list of column names. + + parameters + ---------- + col: string/int, required + The column name or number. + rows: optional + An optional set of row numbers to read. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + res = self.read_columns( + [col], + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + colname = res.dtype.names[0] + data = res[colname] + + self._maybe_trim_strings(data, trim_strings=trim_strings) + return data + + def read_rows( + self, + rows, + vstorage=None, + upper=False, + lower=False, + trim_strings=False, + **keys, + ): + """ + Read the specified rows. + + parameters + ---------- + rows: list,array + A list or array of row indices. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if rows is None: + # we actually want all rows! + return self._read_all() + + if self._info['hdutype'] == ASCII_TBL: + return self.read( + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + + rows, sortind = self._extract_rows(rows) + dtype, offsets, isvar = self.get_rec_dtype(vstorage=vstorage) + + (w,) = np.where(isvar == True) # noqa + has_tbit = self._check_tbit() + + if w.size > 0: + if vstorage is None: + _vstorage = self._vstorage + else: + _vstorage = vstorage + colnums = self._extract_colnums() + return self._read_rec_with_var( + colnums, + rows, + sortind, + dtype, + offsets, + isvar, + _vstorage, + ) + elif has_tbit: + # drop down to read_columns since we can't stuff into a + # contiguous array + colnums = self._extract_colnums() + array = self.read_columns( + colnums, + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + else: + array = np.zeros(rows.size, dtype=dtype) + self._FITS.read_rows_as_rec(self._ext + 1, array, rows, sortind) + + array = self._maybe_decode_fits_ascii_strings_to_unicode_py3(array) + + for colnum, name in enumerate(array.dtype.names): + self._rescale_and_convert_field_inplace( + array, + name, + self._info['colinfo'][colnum]['tscale'], + self._info['colinfo'][colnum]['tzero'], + ) + + if self.lower or lower: + _names_to_lower_if_recarray(array) + elif self.upper or upper: + _names_to_upper_if_recarray(array) + + self._maybe_trim_strings(array, trim_strings=trim_strings) + + return array + + def read_columns( + self, + columns, + rows=None, + vstorage=None, + upper=False, + lower=False, + trim_strings=False, + **keys, + ): + """ + read a subset of columns from this binary table HDU + + By default, all rows are read. Send rows= to select subsets of the + data. Table data are read into a recarray for multiple columns, + plain array for a single column. + + parameters + ---------- + columns: list/array + An optional set of columns to read from table HDUs. Can be string + or number. If a sequence, a recarray is always returned. If a + scalar, an ordinary array is returned. + rows: list/array, optional + An optional list of rows to read from table HDUS. Default is to + read all. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if self._info['hdutype'] == ASCII_TBL: + return self.read( + columns=columns, + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + + # if columns is None, returns all. Guaranteed to be unique and sorted + colnums = self._extract_colnums(columns) + if isinstance(colnums, int): + # scalar sent, don't read as a recarray + return self.read_column( + columns, + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + + # if rows is None still returns None, and is correctly interpreted + # by the reader to mean all + rows, sortind = self._extract_rows(rows) + + # this is the full dtype for all columns + dtype, offsets, isvar = self.get_rec_dtype( + colnums=colnums, vstorage=vstorage + ) + + (w,) = np.where(isvar == True) # noqa + if w.size > 0: + if vstorage is None: + _vstorage = self._vstorage + else: + _vstorage = vstorage + array = self._read_rec_with_var( + colnums, + rows, + sortind, + dtype, + offsets, + isvar, + _vstorage, + ) + else: + if rows is None: + nrows = self._info['nrows'] + else: + nrows = rows.size + + array = np.zeros(nrows, dtype=dtype) + + colnumsp = colnums[:].copy() + colnumsp[:] += 1 + self._FITS.read_columns_as_rec( + self._ext + 1, colnumsp, array, rows, sortind + ) + + array = self._maybe_decode_fits_ascii_strings_to_unicode_py3(array) + + for i in xrange(colnums.size): + colnum = int(colnums[i]) + name = array.dtype.names[i] + self._rescale_and_convert_field_inplace( + array, + name, + self._info['colinfo'][colnum]['tscale'], + self._info['colinfo'][colnum]['tzero'], + ) + + if self._check_tbit(colnums=colnums): + array = self._fix_tbit_dtype(array, colnums) + + if self.lower or lower: + _names_to_lower_if_recarray(array) + elif self.upper or upper: + _names_to_upper_if_recarray(array) + + self._maybe_trim_strings(array, trim_strings=trim_strings) + + return array + + def read_slice( + self, + firstrow, + lastrow, + step=1, + vstorage=None, + lower=False, + upper=False, + trim_strings=False, + **keys, + ): + """ + Read the specified row slice from a table. + + Read all rows between firstrow and lastrow (non-inclusive, as per + python slice notation). Note you must use slice notation for + images, e.g. f[ext][20:30, 40:50] + + parameters + ---------- + firstrow: integer + The first row to read + lastrow: integer + The last row to read, non-inclusive. This follows the python list + slice convention that one does not include the last element. + step: integer, optional + Step between rows, default 1. e.g., if step is 2, skip every other + row. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if self._info['hdutype'] == ASCII_TBL: + rows = np.arange(firstrow, lastrow, step, dtype='i8') + return self.read_ascii( + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + + if self._info['hdutype'] == IMAGE_HDU: + raise ValueError("slices currently only supported for tables") + + maxrow = self._info['nrows'] + if firstrow < 0 or lastrow > maxrow: + raise ValueError( + "slice must specify a sub-range of [%d,%d]" % (0, maxrow) + ) + + dtype, offsets, isvar = self.get_rec_dtype(vstorage=vstorage) + + (w,) = np.where(isvar == True) # noqa + has_tbit = self._check_tbit() + + if w.size > 0: + if vstorage is None: + _vstorage = self._vstorage + else: + _vstorage = vstorage + rows = np.arange(firstrow, lastrow, step, dtype='i8') + sortind = np.arange(rows.size, dtype='i8') + colnums = self._extract_colnums() + array = self._read_rec_with_var( + colnums, rows, sortind, dtype, offsets, isvar, _vstorage + ) + elif has_tbit: + # drop down to read_columns since we can't stuff into a + # contiguous array + colnums = self._extract_colnums() + rows = np.arange(firstrow, lastrow, step, dtype='i8') + array = self.read_columns( + colnums, + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + else: + if step != 1: + rows = np.arange(firstrow, lastrow, step, dtype='i8') + array = self.read(rows=rows) + else: + # no +1 because lastrow is non-inclusive + nrows = lastrow - firstrow + array = np.zeros(nrows, dtype=dtype) + + # only first needs to be +1. This is becuase the c code is + # inclusive + self._FITS.read_as_rec( + self._ext + 1, firstrow + 1, lastrow, array + ) + + array = self._maybe_decode_fits_ascii_strings_to_unicode_py3( + array + ) + + for colnum, name in enumerate(array.dtype.names): + self._rescale_and_convert_field_inplace( + array, + name, + self._info['colinfo'][colnum]['tscale'], + self._info['colinfo'][colnum]['tzero'], + ) + + if self.lower or lower: + _names_to_lower_if_recarray(array) + elif self.upper or upper: + _names_to_upper_if_recarray(array) + + self._maybe_trim_strings(array, trim_strings=trim_strings) + + return array + + def get_rec_dtype(self, colnums=None, vstorage=None, **keys): + """ + Get the dtype for the specified columns + + parameters + ---------- + colnums: integer array, optional + The column numbers, 0 offset + vstorage: string, optional + See docs in read_columns + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if vstorage is None: + _vstorage = self._vstorage + else: + _vstorage = vstorage + + if colnums is None: + colnums = self._extract_colnums() + + descr = [] + isvararray = np.zeros(len(colnums), dtype=bool) + for i, colnum in enumerate(colnums): + dt, isvar = self.get_rec_column_descr(colnum, _vstorage) + descr.append(dt) + isvararray[i] = isvar + dtype = np.dtype(descr) + + offsets = np.zeros(len(colnums), dtype='i8') + for i, n in enumerate(dtype.names): + offsets[i] = dtype.fields[n][1] + return dtype, offsets, isvararray + + def _check_tbit(self, colnums=None, **keys): + """ + Check if one of the columns is a TBIT column + + parameters + ---------- + colnums: integer array, optional + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if colnums is None: + colnums = self._extract_colnums() + + has_tbit = False + for i, colnum in enumerate(colnums): + npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) + if istbit: + has_tbit = True + break + + return has_tbit + + def _fix_tbit_dtype(self, array, colnums): + """ + If necessary, patch up the TBIT to convert to bool array + + parameters + ---------- + array: record array + colnums: column numbers for lookup + """ + descr = array.dtype.descr + for i, colnum in enumerate(colnums): + npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) + if istbit: + coldescr = list(descr[i]) + coldescr[1] = '?' + descr[i] = tuple(coldescr) + + return array.view(descr) + + def _get_simple_dtype_and_shape(self, colnum, rows=None): + """ + When reading a single column, we want the basic data + type and the shape of the array. + + for scalar columns, shape is just nrows, otherwise + it is (nrows, dim1, dim2) + + Note if rows= is sent and only a single row is requested, + the shape will be (dim2,dim2) + """ + + # basic datatype + npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) + info = self._info['colinfo'][colnum] + name = info['name'] + + if rows is None: + nrows = self._info['nrows'] + else: + nrows = rows.size + + shape = None + tdim = info['tdim'] + + shape = _tdim2shape(tdim, name, is_string=(npy_type[0] == 'S')) + if shape is not None: + if nrows > 1: + if not isinstance(shape, tuple): + # vector + shape = (nrows, shape) + else: + # multi-dimensional + shape = tuple([nrows] + list(shape)) + else: + # scalar + shape = nrows + return npy_type, shape + + def get_rec_column_descr(self, colnum, vstorage): + """ + Get a descriptor entry for the specified column. + + parameters + ---------- + colnum: integer + The column number, 0 offset + vstorage: string + See docs in read_columns + """ + npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum) + name = self._info['colinfo'][colnum]['name'] + + if isvar: + if vstorage == 'object': + descr = (name, 'O') + else: + tform = self._info['colinfo'][colnum]['tform'] + max_size = _extract_vararray_max(tform) + + if max_size <= 0: + name = self._info['colinfo'][colnum]['name'] + mess = 'Will read as an object field' + if max_size < 0: + mess = "Column '%s': No maximum size: '%s'. %s" + mess = mess % (name, tform, mess) + warnings.warn(mess, FITSRuntimeWarning) + else: + mess = "Column '%s': Max size is zero: '%s'. %s" + mess = mess % (name, tform, mess) + warnings.warn(mess, FITSRuntimeWarning) + + # we are forced to read this as an object array + return self.get_rec_column_descr(colnum, 'object') + + if npy_type[0] == 'S': + # variable length string columns cannot + # themselves be arrays I don't think + npy_type = 'S%d' % max_size + descr = (name, npy_type) + elif npy_type[0] == 'U': + # variable length string columns cannot + # themselves be arrays I don't think + npy_type = 'U%d' % max_size + descr = (name, npy_type) + else: + descr = (name, npy_type, max_size) + else: + tdim = self._info['colinfo'][colnum]['tdim'] + shape = _tdim2shape( + tdim, + name, + is_string=(npy_type[0] == 'S' or npy_type[0] == 'U'), + ) + if shape is not None: + descr = (name, npy_type, shape) + else: + descr = (name, npy_type) + return descr, isvar + + def _read_rec_with_var( + self, colnums, rows, sortind, dtype, offsets, isvar, vstorage + ): + """ + Read columns from a table into a rec array, including variable length + columns. This is special because, for efficiency, it involves reading + from the main table as normal but skipping the columns in the array + that are variable. Then reading the variable length columns, with + accounting for strides appropriately. + + row and column numbers should be checked before calling this function + """ + + colnumsp = colnums + 1 + if rows is None: + nrows = self._info['nrows'] + else: + nrows = rows.size + array = np.zeros(nrows, dtype=dtype) + + # read from the main table first + (wnotvar,) = np.where(isvar == False) # noqa + if wnotvar.size > 0: + # this will be contiguous (not true for slices) + thesecol = colnumsp[wnotvar] + theseoff = offsets[wnotvar] + self._FITS.read_columns_as_rec_byoffset( + self._ext + 1, + thesecol, + theseoff, + array, + rows, + sortind, + ) + for i in xrange(thesecol.size): + name = array.dtype.names[wnotvar[i]] + colnum = thesecol[i] - 1 + self._rescale_and_convert_field_inplace( + array, + name, + self._info['colinfo'][colnum]['tscale'], + self._info['colinfo'][colnum]['tzero'], + ) + + array = self._maybe_decode_fits_ascii_strings_to_unicode_py3(array) + + # now read the variable length arrays we may be able to speed this up + # by storing directly instead of reading first into a list + (wvar,) = np.where(isvar == True) # noqa + if wvar.size > 0: + # this will be contiguous (not true for slices) + thesecol = colnumsp[wvar] + for i in xrange(thesecol.size): + colnump = thesecol[i] + name = array.dtype.names[wvar[i]] + dlist = self._FITS.read_var_column_as_list( + self._ext + 1, + colnump, + rows, + sortind, + ) + + if isinstance(dlist[0], str) or ( + IS_PY3 and isinstance(dlist[0], bytes) + ): + is_string = True + else: + is_string = False + + if array[name].dtype.descr[0][1][1] == 'O': + # storing in object array + # get references to each, no copy made + for irow, item in enumerate(dlist): + if sortind is not None: + irow = sortind[irow] + if IS_PY3 and isinstance(item, bytes): + item = item.decode('ascii') + array[name][irow] = item + else: + for irow, item in enumerate(dlist): + if sortind is not None: + irow = sortind[irow] + if IS_PY3 and isinstance(item, bytes): + item = item.decode('ascii') + + if is_string: + array[name][irow] = item + else: + ncopy = len(item) + + if IS_PY3: + ts = array[name].dtype.descr[0][1][1] + if ts != 'S' and ts != 'U': + array[name][irow][0:ncopy] = item[:] + else: + array[name][irow] = item + else: + array[name][irow][0:ncopy] = item[:] + + return array + + def _extract_rows(self, rows, sort=False): + """ + Extract an array of rows from an input scalar or sequence + """ + if rows is not None: + rows = np.array(rows, ndmin=1, copy=copy_if_needed, dtype='i8') + if sort: + rows = np.unique(rows) + return rows, None + + # returns unique, sorted. Force i8 for 32-bit systems + sortind = np.array(rows.argsort(), dtype='i8', copy=copy_if_needed) + + maxrow = self._info['nrows'] - 1 + if rows.size > 0: + firstrow = rows[sortind[0]] + lastrow = rows[sortind[-1]] + + if len(rows) > 0 and (firstrow < 0 or lastrow > maxrow): + raise ValueError("rows must be in [%d,%d]" % (0, maxrow)) + else: + sortind = None + + return rows, sortind + + def _process_slice(self, arg): + """ + process the input slice for use calling the C code + """ + start = arg.start + stop = arg.stop + step = arg.step + + nrows = self._info['nrows'] + if step is None: + step = 1 + if start is None: + start = 0 + if stop is None: + stop = nrows + + if start < 0: + start = nrows + start + if start < 0: + raise IndexError("Index out of bounds") + + if stop < 0: + stop = nrows + start + 1 + + if stop < start: + # will return an empty struct + stop = start + + if stop > nrows: + stop = nrows + return slice(start, stop, step) + + def _slice2rows(self, start, stop, step=None): + """ + Convert a slice to an explicit array of rows + """ + nrows = self._info['nrows'] + if start is None: + start = 0 + if stop is None: + stop = nrows + if step is None: + step = 1 + + tstart = self._fix_range(start) + tstop = self._fix_range(stop) + if tstart == 0 and tstop == nrows and step is None: + # this is faster: if all fields are also requested, then a + # single fread will be done + return None + if stop < start: + raise ValueError("start is greater than stop in slice") + return np.arange(tstart, tstop, step, dtype='i8') + + def _fix_range(self, num, isslice=True): + """ + Ensure the input is within range. + + If el=True, then don't treat as a slice element + """ + + nrows = self._info['nrows'] + if isslice: + # include the end + if num < 0: + num = nrows + (1 + num) + elif num > nrows: + num = nrows + else: + # single element + if num < 0: + num = nrows + num + elif num > (nrows - 1): + num = nrows - 1 + + return num + + def _rescale_and_convert_field_inplace(self, array, name, scale, zero): + """ + Apply fits scalings. Also, convert bool to proper + numpy boolean values + """ + self._rescale_array(array[name], scale, zero) + if array[name].dtype == bool: + array[name] = self._convert_bool_array(array[name]) + return array + + def _rescale_and_convert(self, array, scale, zero, name=None): + """ + Apply fits scalings. Also, convert bool to proper + numpy boolean values + """ + self._rescale_array(array, scale, zero) + if array.dtype == bool: + array = self._convert_bool_array(array) + + return array + + def _rescale_array(self, array, scale, zero): + """ + Scale the input array + """ + if scale != 1.0: + sval = np.array(scale, dtype=array.dtype) + array *= sval + if zero != 0.0: + zval = np.array(zero, dtype=array.dtype) + array += zval + + def _maybe_trim_strings(self, array, trim_strings=False, **keys): + """ + if requested, trim trailing white space from + all string fields in the input array + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if self.trim_strings or trim_strings: + _trim_strings(array) + + def _maybe_decode_fits_ascii_strings_to_unicode_py3(self, array): + if IS_PY3: + do_conversion = False + new_dt = [] + for _dt in array.dtype.descr: + if 'S' in _dt[1]: + do_conversion = True + if len(_dt) == 3: + new_dt.append( + ( + _dt[0], + _dt[1].replace('S', 'U').replace('|', ''), + _dt[2], + ) + ) + else: + new_dt.append( + (_dt[0], _dt[1].replace('S', 'U').replace('|', '')) + ) + else: + new_dt.append(_dt) + if do_conversion: + array = array.astype(new_dt, copy=copy_if_needed) + return array + + def _convert_bool_array(self, array): + """ + cfitsio reads as characters 'T' and 'F' -- convert to real boolean + If input is a fits bool, convert to numpy boolean + """ + + output = (array.view(np.int8) == ord('T')).astype(bool) + return output + + def _get_tbl_numpy_dtype(self, colnum, include_endianness=True): + """ + Get numpy type for the input column + """ + table_type = self._info['hdutype'] + table_type_string = _hdu_type_map[table_type] + try: + ftype = self._info['colinfo'][colnum]['eqtype'] + if table_type == ASCII_TBL: + npy_type = _table_fits2npy_ascii[abs(ftype)] + else: + npy_type = _table_fits2npy[abs(ftype)] + except KeyError: + raise KeyError( + "unsupported %s fits data " + "type: %d" % (table_type_string, ftype) + ) + + istbit = False + if ftype == 1: + istbit = True + + isvar = False + if ftype < 0: + isvar = True + if include_endianness: + # if binary we will read the big endian bytes directly, + # if ascii we read into native byte order + if table_type == ASCII_TBL: + addstr = '' + else: + addstr = '>' + if npy_type not in ['u1', 'i1', 'S', 'U']: + npy_type = addstr + npy_type + + if npy_type == 'S': + width = self._info['colinfo'][colnum]['width'] + npy_type = 'S%d' % width + elif npy_type == 'U': + width = self._info['colinfo'][colnum]['width'] + npy_type = 'U%d' % width + + return npy_type, isvar, istbit + + def _process_args_as_rows_or_columns(self, arg, unpack=False): + """ + We must be able to interpret the args as as either a column name or + row number, or sequences thereof. Numpy arrays and slices are also + fine. + + Examples: + 'field' + 35 + [35,55,86] + ['f1',f2',...] + Can also be tuples or arrays. + """ + + flags = set() + # + if isinstance(arg, (tuple, list, np.ndarray)): + # a sequence was entered + if isstring(arg[0]): + result = arg + else: + result = arg + flags.add('isrows') + elif isstring(arg): + # a single string was entered + result = arg + elif isinstance(arg, slice): + if unpack: + flags.add('isrows') + result = self._slice2rows(arg.start, arg.stop, arg.step) + else: + flags.add('isrows') + flags.add('isslice') + result = self._process_slice(arg) + else: + # a single object was entered. + # Probably should apply some more checking on this + result = arg + flags.add('isrows') + if np.ndim(arg) == 0: + flags.add('isscalar') + + return result, flags + + def _read_var_column(self, colnum, rows, sortind, vstorage): + """ + + first read as a list of arrays, then copy into either a fixed length + array or an array of objects, depending on vstorage. + + """ + + if IS_PY3: + stype = bytes + else: + stype = str + + dlist = self._FITS.read_var_column_as_list( + self._ext + 1, + colnum + 1, + rows, + sortind, + ) + + if vstorage == 'fixed': + tform = self._info['colinfo'][colnum]['tform'] + max_size = _extract_vararray_max(tform) + + if max_size <= 0: + name = self._info['colinfo'][colnum]['name'] + mess = 'Will read as an object field' + if max_size < 0: + mess = "Column '%s': No maximum size: '%s'. %s" + mess = mess % (name, tform, mess) + warnings.warn(mess, FITSRuntimeWarning) + else: + mess = "Column '%s': Max size is zero: '%s'. %s" + mess = mess % (name, tform, mess) + warnings.warn(mess, FITSRuntimeWarning) + + # we are forced to read this as an object array + return self._read_var_column(colnum, rows, 'object') + + if isinstance(dlist[0], stype): + descr = 'S%d' % max_size + array = np.fromiter(dlist, descr) + if IS_PY3: + array = array.astype('U', copy=copy_if_needed) + else: + descr = dlist[0].dtype.str + array = np.zeros((len(dlist), max_size), dtype=descr) + + for irow, item in enumerate(dlist): + if sortind is not None: + irow = sortind[irow] + ncopy = len(item) + array[irow, 0:ncopy] = item[:] + else: + array = np.zeros(len(dlist), dtype='O') + for irow, item in enumerate(dlist): + if sortind is not None: + irow = sortind[irow] + + if IS_PY3 and isinstance(item, bytes): + item = item.decode('ascii') + array[irow] = item + + return array + + def _extract_colnums(self, columns=None): + """ + Extract an array of columns from the input + """ + if columns is None: + return np.arange(self._info["ncol"], dtype='i8') + + if not isinstance(columns, (tuple, list, np.ndarray)): + # is a scalar + return self._extract_colnum(columns) + + colnums = np.zeros(len(columns), dtype='i8') + for i in xrange(colnums.size): + colnums[i] = self._extract_colnum(columns[i]) + + # returns unique sorted + colnums = np.unique(colnums) + return colnums + + def _extract_colnum(self, col): + """ + Get the column number for the input column + """ + if isinteger(col): + colnum = col + + if (colnum < 0) or (colnum > (self._info["ncol"] - 1)): + raise ValueError( + "column number should be in [0,%d]" + % (self._info["ncol"] - 1) + ) + else: + colstr = mks(col) + try: + if self.case_sensitive: + mess = "column name '%s' not found (case sensitive)" % col + colnum = self._info["colnames"].index(colstr) + else: + mess = ( + "column name '%s' not found (case insensitive)" % col + ) + colnum = self._info["colnames_lower"].index(colstr.lower()) + except ValueError: + raise ValueError(mess) + return int(colnum) + + def _update_info(self): + """ + Call parent method and make sure this is in fact a + table HDU. Set some convenience data. + """ + super(TableHDU, self)._update_info() + if self._info['hdutype'] == IMAGE_HDU: + mess = "Extension %s is not a Table HDU" % self.ext + raise ValueError(mess) + if 'colinfo' in self._info: + self._info["colnames"] = [i['name'] for i in self._info['colinfo']] + self._info["colnames_lower"] = [ + i['name'].lower() for i in self._info['colinfo'] + ] + self._info["ncol"] = len(self._info["colnames"]) + + def __getitem__(self, arg): + """ + Get data from a table using python [] notation. + + You can use [] to extract column and row subsets, or read everything. + The notation is essentially the same as numpy [] notation, except that + a sequence of column names may also be given. Examples reading from + "filename", extension "ext" + + fits=fitsio.FITS(filename) + fits[ext][:] + fits[ext][2] # returns a scalar + fits[ext][2:5] + fits[ext][200:235:2] + fits[ext][rows] + fits[ext][cols][rows] + + Note data are only read once the rows are specified. + + Note you can only read variable length arrays the default way, + using this function, so set it as you want on construction. + + This function is used for ascii tables as well + """ + + res, flags = self._process_args_as_rows_or_columns(arg) + + if 'isrows' in flags: + # rows were entered: read all columns + if 'isslice' in flags: + array = self.read_slice(res.start, res.stop, res.step) + else: + # will also get here if slice is entered but this + # is an ascii table + array = self.read(rows=res) + else: + return TableColumnSubset(self, res) + + if self.lower: + _names_to_lower_if_recarray(array) + elif self.upper: + _names_to_upper_if_recarray(array) + + self._maybe_trim_strings(array) + + if 'isscalar' in flags: + assert array.shape[0] == 1 + array = array[0] + return array + + def __iter__(self): + """ + Get an iterator for a table + + e.g. + f=fitsio.FITS(fname) + hdu1 = f[1] + for row in hdu1: + ... + """ + + # always start with first row + self._iter_row = 0 + + # for iterating we must assume the number of rows will not change + self._iter_nrows = self.get_nrows() + + self._buffer_iter_rows(0) + return self + + def next(self): + """ + get the next row when iterating + + e.g. + f=fitsio.FITS(fname) + hdu1 = f[1] + for row in hdu1: + ... + + By default read one row at a time. Send iter_row_buffer to get a more + efficient buffering. + """ + return self._get_next_buffered_row() + + __next__ = next + + def _get_next_buffered_row(self): + """ + Get the next row for iteration. + """ + if self._iter_row == self._iter_nrows: + raise StopIteration + + if self._row_buffer_index >= self._iter_row_buffer: + self._buffer_iter_rows(self._iter_row) + + data = self._row_buffer[self._row_buffer_index] + self._iter_row += 1 + self._row_buffer_index += 1 + return data + + def _buffer_iter_rows(self, start): + """ + Read in the buffer for iteration + """ + self._row_buffer = self[start : start + self._iter_row_buffer] + + # start back at the front of the buffer + self._row_buffer_index = 0 + + def __repr__(self): + """ + textual representation for some metadata + """ + text, spacing = self._get_repr_list() + + text.append('%srows: %d' % (spacing, self._info['nrows'])) + text.append('%scolumn info:' % spacing) + + cspacing = ' ' * 4 + nspace = 4 + nname = 15 + ntype = 6 + format = cspacing + "%-" + str(nname) + "s %" + str(ntype) + "s %s" + pformat = ( + cspacing + + "%-" + + str(nname) + + "s\n %" + + str(nspace + nname + ntype) + + "s %s" + ) + + for colnum, c in enumerate(self._info['colinfo']): + if len(c['name']) > nname: + f = pformat + else: + f = format + + dt, isvar, istbit = self._get_tbl_numpy_dtype( + colnum, include_endianness=False + ) + if isvar: + tform = self._info['colinfo'][colnum]['tform'] + if dt[0] == 'S': + dt = 'S0' + dimstr = 'vstring[%d]' % _extract_vararray_max(tform) + else: + dimstr = 'varray[%s]' % _extract_vararray_max(tform) + else: + if dt[0] == 'S': + is_string = True + else: + is_string = False + dimstr = _get_col_dimstr(c['tdim'], is_string=is_string) + + s = f % (c['name'], dt, dimstr) + text.append(s) + + text = '\n'.join(text) + return text + + +class AsciiTableHDU(TableHDU): + def read( + self, + rows=None, + columns=None, + vstorage=None, + upper=False, + lower=False, + trim_strings=False, + **keys, + ): + """ + read a data from an ascii table HDU + + By default, all rows are read. Send rows= to select subsets of the + data. Table data are read into a recarray for multiple columns, + plain array for a single column. + + parameters + ---------- + columns: list/array + An optional set of columns to read from table HDUs. Can be string + or number. If a sequence, a recarray is always returned. If a + scalar, an ordinary array is returned. + rows: list/array, optional + An optional list of rows to read from table HDUS. Default is to + read all. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + # if columns is None, returns all. Guaranteed to be unique and sorted + colnums = self._extract_colnums(columns) + if isinstance(colnums, int): + # scalar sent, don't read as a recarray + return self.read_column( + columns, + rows=rows, + vstorage=vstorage, + upper=upper, + lower=lower, + trim_strings=trim_strings, + ) + + rows, sortind = self._extract_rows(rows) + if rows is None: + nrows = self._info['nrows'] + else: + nrows = rows.size + + # if rows is None still returns None, and is correctly interpreted + # by the reader to mean all + rows, sortind = self._extract_rows(rows) + + # this is the full dtype for all columns + dtype, offsets, isvar = self.get_rec_dtype( + colnums=colnums, vstorage=vstorage + ) + array = np.zeros(nrows, dtype=dtype) + + # note reading into existing data + (wnotvar,) = np.where(isvar == False) # noqa + if wnotvar.size > 0: + for i in wnotvar: + colnum = colnums[i] + name = array.dtype.names[i] + a = array[name].copy() + self._FITS.read_column( + self._ext + 1, colnum + 1, a, rows, sortind + ) + array[name] = a + del a + + array = self._maybe_decode_fits_ascii_strings_to_unicode_py3(array) + + (wvar,) = np.where(isvar == True) # noqa + if wvar.size > 0: + for i in wvar: + colnum = colnums[i] + name = array.dtype.names[i] + dlist = self._FITS.read_var_column_as_list( + self._ext + 1, + colnum + 1, + rows, + sortind, + ) + if isinstance(dlist[0], str) or ( + IS_PY3 and isinstance(dlist[0], bytes) + ): + is_string = True + else: + is_string = False + + if array[name].dtype.descr[0][1][1] == 'O': + # storing in object array + # get references to each, no copy made + for irow, item in enumerate(dlist): + if sortind is not None: + irow = sortind[irow] + if IS_PY3 and isinstance(item, bytes): + item = item.decode('ascii') + array[name][irow] = item + else: + for irow, item in enumerate(dlist): + if sortind is not None: + irow = sortind[irow] + if IS_PY3 and isinstance(item, bytes): + item = item.decode('ascii') + if is_string: + array[name][irow] = item + else: + ncopy = len(item) + array[name][irow][0:ncopy] = item[:] + + if self.lower or lower: + _names_to_lower_if_recarray(array) + elif self.upper or upper: + _names_to_upper_if_recarray(array) + + self._maybe_trim_strings(array, trim_strings=trim_strings) + + return array + + read_ascii = read + + +class TableColumnSubset(object): + """ + + A class representing a subset of the the columns on disk. When called + with .read() or [ rows ] the data are read from disk. + + Useful because subsets can be passed around to functions, or chained + with a row selection. + + This class is returned when using [ ] notation to specify fields in a + TableHDU class + + fits = fitsio.FITS(fname) + colsub = fits[ext][field_list] + + returns a TableColumnSubset object. To read rows: + + data = fits[ext][field_list][row_list] + + colsub = fits[ext][field_list] + data = colsub[row_list] + data = colsub.read(rows=row_list) + + to read all, use .read() with no args or [:] + """ + + def __init__(self, fitshdu, columns): + """ + Input is the FITS instance and a list of column names. + """ + + self.columns = columns + if isstring(columns) or isinteger(columns): + # this is to check if it exists + self.colnums = [fitshdu._extract_colnum(columns)] + + self.is_scalar = True + self.columns_list = [columns] + else: + # this is to check if it exists + self.colnums = fitshdu._extract_colnums(columns) + + self.is_scalar = False + self.columns_list = columns + + self.fitshdu = fitshdu + + def read( + self, + columns=None, + rows=None, + vstorage=None, + lower=False, + upper=False, + trim_strings=False, + **keys, + ): + """ + Read the data from disk and return as a numpy array + + parameters + ---------- + columns: list/array, optional + An optional set of columns to read from table HDUs. Can be string + or number. If a sequence, a recarray is always returned. If a + scalar, an ordinary array is returned. + rows: optional + An optional list of rows to read from table HDUS. Default is to + read all. + vstorage: string, optional + Over-ride the default method to store variable length columns. Can + be 'fixed' or 'object'. See docs on fitsio.FITS for details. + lower: bool, optional + If True, force all columns names to lower case in output. Will over + ride the lower= keyword from construction. + upper: bool, optional + If True, force all columns names to upper case in output. Will over + ride the lower= keyword from construction. + trim_strings: bool, optional + If True, trim trailing spaces from strings. Will over-ride the + trim_strings= keyword from constructor. + """ + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if self.is_scalar: + data = self.fitshdu.read_column( + self.columns, + rows=rows, + vstorage=vstorage, + lower=lower, + upper=upper, + trim_strings=trim_strings, + ) + else: + if columns is None: + c = self.columns + else: + c = columns + data = self.fitshdu.read( + columns=c, + rows=rows, + vstorage=vstorage, + lower=lower, + upper=upper, + trim_strings=trim_strings, + ) + + return data + + def __getitem__(self, arg): + """ + If columns are sent, then the columns will just get reset and + we'll return a new object + + If rows are sent, they are read and the result returned. + """ + + # we have to unpack the rows if we are reading a subset + # of the columns because our slice operator only works + # on whole rows. We could allow rows= keyword to + # be a slice... + + res, flags = self.fitshdu._process_args_as_rows_or_columns( + arg, unpack=True + ) + if 'isrows' in flags: + # rows was entered: read all current column subset + array = self.read(rows=res) + if 'isscalar' in flags: + assert array.shape[0] == 1 + array = array[0] + return array + else: + # columns was entered. Return a subset objects + return TableColumnSubset(self.fitshdu, columns=res) + + def __repr__(self): + """ + Representation for TableColumnSubset + """ + spacing = ' ' * 2 + cspacing = ' ' * 4 + + hdu = self.fitshdu + info = self.fitshdu._info + colinfo = info['colinfo'] + + text = [] + text.append("%sfile: %s" % (spacing, hdu._filename)) + text.append("%sextension: %d" % (spacing, info['hdunum'] - 1)) + text.append("%stype: %s" % (spacing, _hdu_type_map[info['hdutype']])) + text.append('%srows: %d' % (spacing, info['nrows'])) + text.append("%scolumn subset:" % spacing) + + cspacing = ' ' * 4 + nspace = 4 + nname = 15 + ntype = 6 + format = cspacing + "%-" + str(nname) + "s %" + str(ntype) + "s %s" + pformat = ( + cspacing + + "%-" + + str(nname) + + "s\n %" + + str(nspace + nname + ntype) + + "s %s" + ) + + for colnum in self.colnums: + cinfo = colinfo[colnum] + + if len(cinfo['name']) > nname: + f = pformat + else: + f = format + + dt, isvar, istbit = hdu._get_tbl_numpy_dtype( + colnum, include_endianness=False + ) + if isvar: + tform = cinfo['tform'] + if dt[0] == 'S': + dt = 'S0' + dimstr = 'vstring[%d]' % _extract_vararray_max(tform) + else: + dimstr = 'varray[%s]' % _extract_vararray_max(tform) + else: + dimstr = _get_col_dimstr(cinfo['tdim']) + + s = f % (cinfo['name'], dt, dimstr) + text.append(s) + + s = "\n".join(text) + return s + + +def _tdim2shape(tdim, name, is_string=False): + shape = None + if tdim is None: + raise ValueError("field '%s' has malformed TDIM" % name) + + if len(tdim) > 1 or tdim[0] > 1: + if is_string: + shape = list(reversed(tdim[1:])) + else: + shape = list(reversed(tdim)) + + if len(shape) == 1: + shape = shape[0] + else: + shape = tuple(shape) + + return shape + + +def _names_to_lower_if_recarray(data): + if data.dtype.names is not None: + data.dtype.names = [n.lower() for n in data.dtype.names] + + +def _names_to_upper_if_recarray(data): + if data.dtype.names is not None: + data.dtype.names = [n.upper() for n in data.dtype.names] + + +def _trim_strings(data): + names = data.dtype.names + if names is not None: + # run through each field separately + for n in names: + if data[n].dtype.descr[0][1][1] in ['S', 'U']: + data[n] = np.char.rstrip(data[n]) + else: + if data.dtype.descr[0][1][1] in ['S', 'U']: + data[:] = np.char.rstrip(data[:]) + + +def _extract_vararray_max(tform): + """ + Extract number from PX(number) + """ + first = tform.find('(') + last = tform.rfind(')') + + if first == -1 or last == -1: + # no max length specified + return -1 + + maxnum = int(tform[first + 1 : last]) + return maxnum + + +def _get_col_dimstr(tdim, is_string=False): + """ + not for variable length + """ + dimstr = '' + if tdim is None: + dimstr = 'array[bad TDIM]' + else: + if is_string: + if len(tdim) > 1: + dimstr = [str(d) for d in tdim[1:]] + else: + if len(tdim) > 1 or tdim[0] > 1: + dimstr = [str(d) for d in tdim] + if dimstr != '': + dimstr = ','.join(dimstr) + dimstr = 'array[%s]' % dimstr + + return dimstr + + +# no support yet for complex +# all strings are read as bytes for python3 and then decoded to unicode +_table_fits2npy = { + 1: 'i1', + 11: 'u1', + 12: 'i1', + # logical. Note pyfits uses this for i1, + # cfitsio casts to char* + 14: 'b1', + 16: 'S', + 20: 'u2', + 21: 'i2', + 30: 'u4', # 30=TUINT + 31: 'i4', # 31=TINT + 40: 'u4', # 40=TULONG + 41: 'i4', # 41=TLONG + 42: 'f4', + 80: 'u8', # 80=TULONGLON + 81: 'i8', + 82: 'f8', + 83: 'c8', # TCOMPLEX + 163: 'c16', +} # TDBLCOMPLEX + +# cfitsio returns only types f8, i4 and strings for column types. in order to +# avoid data loss, we always use i8 for integer types +# all strings are read as bytes for python3 and then decoded to unicode +_table_fits2npy_ascii = { + 16: 'S', + 31: 'i8', # listed as TINT, reading as i8 + 41: 'i8', # listed as TLONG, reading as i8 + 81: 'i8', + 21: 'i4', # listed as TSHORT, reading as i4 + 42: 'f8', # listed as TFLOAT, reading as f8 + 82: 'f8', +} + +# for TFORM +_table_npy2fits_form = { + 'b1': 'L', + 'u1': 'B', + 'i1': 'S', # gets converted to unsigned + 'S': 'A', + 'U': 'A', + 'u2': 'U', # gets converted to signed + 'i2': 'I', + 'u4': 'V', # gets converted to signed + 'i4': 'J', + 'i8': 'K', + 'u8': 'W', + 'f4': 'E', + 'f8': 'D', + 'c8': 'C', + 'c16': 'M', +} + +# from mrdfits; note G gets turned into E +# types= ['A', 'I', 'L', 'B', 'F', 'D', 'C', 'M', 'K'] +# formats=['A1', 'I6', 'I10', 'I4', 'G15.9','G23.17', 'G15.9', 'G23.17', +# 'I20'] + +_table_npy2fits_form_ascii = { + 'S': 'A1', # Need to add max here + 'U': 'A1', # Need to add max here + 'i2': 'I7', # I + 'i4': 'I12', # ?? + # 'i8':'I21', # K # i8 aren't supported + # 'f4':'E15.7', # F + # F We must write as f8 since we can only + # read as f8 + 'f4': 'E26.17', + # D 25.16 looks right, but this is recommended + 'f8': 'E26.17', +} + + +def _npy2fits(d, table_type='binary', write_bitcols=False): + """ + d is the full element from the descr + """ + npy_dtype = d[1][1:] + if npy_dtype[0] == 'S' or npy_dtype[0] == 'U': + name, form, dim = _npy_string2fits(d, table_type=table_type) + else: + name, form, dim = _npy_num2fits( + d, table_type=table_type, write_bitcols=write_bitcols + ) + + return name, form, dim + + +def _npy_num2fits(d, table_type='binary', write_bitcols=False): + """ + d is the full element from the descr + + For vector,array columns the form is the total counts + followed by the code. + + For array columns with dimension greater than 1, the dim is set to + (dim1, dim2, ...) + So it is treated like an extra dimension + + """ + + dim = None + + name = d[0] + + npy_dtype = d[1][1:] + if npy_dtype[0] == 'S' or npy_dtype[0] == 'U': + raise ValueError("got S or U type: use _npy_string2fits") + + if table_type == 'binary': + if npy_dtype not in _table_npy2fits_form: + raise ValueError( + "unsupported type '%s' for binary tables" % npy_dtype + ) + else: + if npy_dtype not in _table_npy2fits_form_ascii: + raise ValueError( + "unsupported type '%s' for ascii tables" % npy_dtype + ) + + if table_type == 'binary': + form = _table_npy2fits_form[npy_dtype] + else: + form = _table_npy2fits_form_ascii[npy_dtype] + + # now the dimensions + if len(d) > 2: + if table_type == 'ascii': + raise ValueError( + "Ascii table columns must be scalar, got %s" % str(d) + ) + + if write_bitcols and npy_dtype == 'b1': + # multi-dimensional boolean + form = 'X' + + # Note, depending on numpy version, even 1-d can be a tuple + if isinstance(d[2], tuple): + count = reduce(lambda x, y: x * y, d[2]) + form = '%d%s' % (count, form) + + if len(d[2]) > 1: + # this is multi-dimensional array column. the form + # should be total elements followed by A + dim = list(reversed(d[2])) + dim = [str(e) for e in dim] + dim = '(' + ','.join(dim) + ')' + else: + # this is a vector (1d array) column + count = d[2] + form = '%d%s' % (count, form) + + return name, form, dim + + +def _npy_string2fits(d, table_type='binary'): + """ + d is the full element from the descr + + form for strings is the total number of bytes followed by A. Thus + for vector or array columns it is the size of the string times the + total number of elements in the array. + + Then the dim is set to + (sizeofeachstring, dim1, dim2, ...) + So it is treated like an extra dimension + + """ + + dim = None + + name = d[0] + + npy_dtype = d[1][1:] + if npy_dtype[0] != 'S' and npy_dtype[0] != 'U': + raise ValueError("expected S or U type, got %s" % npy_dtype[0]) + + # get the size of each string + string_size_str = npy_dtype[1:] + string_size = int(string_size_str) + + if string_size <= 0: + raise ValueError( + 'string sizes must be > 0, got %s for field %s' % (npy_dtype, name) + ) + + # now the dimensions + if len(d) == 2: + if table_type == 'ascii': + form = 'A' + string_size_str + else: + form = string_size_str + 'A' + else: + if table_type == 'ascii': + raise ValueError( + "Ascii table columns must be scalar, got %s" % str(d) + ) + if isinstance(d[2], tuple): + # this is an array column. the form + # should be total elements followed by A + # count = 1 + # count = [count*el for el in d[2]] + count = reduce(lambda x, y: x * y, d[2]) + count = string_size * count + form = '%dA' % count + + if len(d[2]) == 1 and d[2][0] == 1: + # string vec length 1 are written as scalars + pass + else: + dim = list(reversed(d[2])) + # dim = d[2] + dim = [string_size_str] + [str(e) for e in dim] + dim = '(' + ','.join(dim) + ')' + else: + # this is a vector (1d array) column + count = string_size * d[2] + form = '%dA' % count + + # will have to do tests to see if this is the right order + dim = [string_size_str, str(d[2])] + dim = '(' + ','.join(dim) + ')' + + return name, form, dim diff --git a/fitsio/header.py b/fitsio/header.py new file mode 100644 index 0000000..cd9275f --- /dev/null +++ b/fitsio/header.py @@ -0,0 +1,781 @@ +""" +header classes for fitslib, part of the fitsio package. + +See the main docs at https://github.com/esheldon/fitsio + + Copyright (C) 2011 Erin Sheldon, BNL. erin dot sheldon at gmail dot com + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" + +from __future__ import with_statement, print_function +import warnings + +from . import _fitsio_wrap +from .util import isstring, FITSRuntimeWarning, IS_PY3 + +# for python3 compat +if IS_PY3: + xrange = range + +TYP_STRUC_KEY = 10 +TYP_CMPRS_KEY = 20 +TYP_SCAL_KEY = 30 +TYP_NULL_KEY = 40 +TYP_DIM_KEY = 50 +TYP_RANG_KEY = 60 +TYP_UNIT_KEY = 70 +TYP_DISP_KEY = 80 +TYP_HDUID_KEY = 90 +TYP_CKSUM_KEY = 100 +TYP_WCS_KEY = 110 +TYP_REFSYS_KEY = 120 +TYP_COMM_KEY = 130 +TYP_CONT_KEY = 140 +TYP_USER_KEY = 150 + + +class FITSHDR(object): + """ + A class representing a FITS header. + + parameters + ---------- + record_list: optional + A list of dicts, or dict, or another FITSHDR + - list of dictionaries containing 'name','value' and optionally + a 'comment' field; the order is preserved. + - a dictionary of keyword-value pairs; no comments are written + in this case, and the order is arbitrary. + - another FITSHDR object; the order is preserved. + + examples: + + hdr=FITSHDR() + + # set a simple value + hdr['blah'] = 35 + + # set from a dict to include a comment. + rec={'name':'fromdict', 'value':3, 'comment':'my comment'} + hdr.add_record(rec) + + # can do the same with a full FITSRecord + rec=FITSRecord( {'name':'temp', 'value':35, 'comment':'temp in C'} ) + hdr.add_record(rec) + + # in the above, the record is replaced if one with the same name + # exists, except for COMMENT and HISTORY, which can exist as + # duplicates + + # print the header + print(hdr) + + # print a single record + print(hdr['fromdict']) + + + # can also set from a card + hdr.add_record('test = 77') + # using a FITSRecord object (internally uses FITSCard) + card=FITSRecord('test = 77') + hdr.add_record(card) + + # can also construct with a record list + recs=[{'name':'test', 'value':35, 'comment':'a comment'}, + {'name':'blah', 'value':'some string'}] + hdr=FITSHDR(recs) + + # if you have no comments, you can construct with a simple dict + recs={'day':'saturday', + 'telescope':'blanco'} + hdr=FITSHDR(recs) + + """ + + def __init__(self, record_list=None): + self._record_list = [] + self._record_map = {} + self._index_map = {} + + if isinstance(record_list, FITSHDR): + for r in record_list.records(): + self.add_record(r) + elif isinstance(record_list, dict): + for k in record_list: + r = {'name': k, 'value': record_list[k]} + self.add_record(r) + elif isinstance(record_list, list): + for r in record_list: + self.add_record(r) + elif record_list is not None: + raise ValueError("expected a dict or list of dicts or FITSHDR") + + def add_record(self, record_in): + """ + Add a new record. Strip quotes from around strings. + + This will over-write if the key already exists, except + for COMMENT and HISTORY fields + + parameters + ----------- + record: + The record, either a dict or a header card string + or a FITSRecord or FITSCard + """ + if ( + isinstance(record_in, dict) + and 'name' in record_in + and 'value' in record_in + ): + record = {} + record.update(record_in) + else: + record = FITSRecord(record_in) + + # only append when this name already exists if it is + # a comment or history field, otherwise simply over-write + key = record['name'] + if key is not None: + key = key.upper() + + key_exists = key in self._record_map + + if not key_exists or key in ('COMMENT', 'HISTORY', 'CONTINUE', None): + # append new record + self._record_list.append(record) + index = len(self._record_list) - 1 + self._index_map[key] = index + else: + # over-write existing + index = self._index_map[key] + self._record_list[index] = record + + self._record_map[key] = record + + def _add_to_map(self, record): + key = record['name'].upper() + self._record_map[key] = record + + def get_comment(self, item): + """ + Get the comment for the requested entry + """ + key = item.upper() + if key not in self._record_map: + raise KeyError("unknown record: %s" % key) + + if 'comment' not in self._record_map[key]: + return None + else: + return self._record_map[key]['comment'] + + def records(self): + """ + Return the list of full records as a list of dictionaries. + """ + return self._record_list + + def keys(self): + """ + Return a copy of the current key list. + """ + return [e['name'] for e in self._record_list] + + def delete(self, name): + """ + Delete the specified entry if it exists. + """ + if isinstance(name, (list, tuple)): + for xx in name: + self.delete(xx) + else: + if name in self._record_map: + del self._record_map[name] + # Store current index value + cur_index = self._index_map[name] + # Delete in index map + del self._index_map[name] + self._record_list = [ + r for r in self._record_list if r['name'] != name + ] + + # Change index map for superior indexes, only + for k, v in self._index_map.items(): + if v > cur_index: + self._index_map[k] = v - 1 + + def clean(self, is_table=False): + """ + Remove reserved keywords from the header. + + These are keywords that the fits writer must write in order + to maintain consistency between header and data. + + keywords + -------- + is_table: bool, optional + Set True if this is a table, so extra keywords will be cleaned + """ + + rmnames = [ + 'SIMPLE', + 'EXTEND', + 'XTENSION', + 'BITPIX', + 'PCOUNT', + 'GCOUNT', + 'THEAP', + 'EXTNAME', + # 'BLANK', + 'ZQUANTIZ', + 'ZDITHER0', + 'ZIMAGE', + 'ZCMPTYPE', + 'ZSIMPLE', + 'ZTENSION', + 'ZPCOUNT', + 'ZGCOUNT', + 'ZBITPIX', + 'ZEXTEND', + # 'FZTILELN','FZALGOR', + 'CHECKSUM', + 'DATASUM', + ] + + if is_table: + # these are not allowed in tables + rmnames += [ + 'BUNIT', + 'BSCALE', + 'BZERO', + ] + + self.delete(rmnames) + + r = self._record_map.get('NAXIS', None) + if r is not None: + naxis = int(r['value']) + self.delete('NAXIS') + + rmnames = ['NAXIS%d' % i for i in xrange(1, naxis + 1)] + self.delete(rmnames) + + r = self._record_map.get('ZNAXIS', None) + self.delete('ZNAXIS') + if r is not None: + znaxis = int(r['value']) + + rmnames = ['ZNAXIS%d' % i for i in xrange(1, znaxis + 1)] + self.delete(rmnames) + rmnames = ['ZTILE%d' % i for i in xrange(1, znaxis + 1)] + self.delete(rmnames) + rmnames = ['ZNAME%d' % i for i in xrange(1, znaxis + 1)] + self.delete(rmnames) + rmnames = ['ZVAL%d' % i for i in xrange(1, znaxis + 1)] + self.delete(rmnames) + + r = self._record_map.get('TFIELDS', None) + if r is not None: + tfields = int(r['value']) + self.delete('TFIELDS') + + if tfields > 0: + nbase = [ + 'TFORM', + 'TTYPE', + 'TDIM', + 'TUNIT', + 'TSCAL', + 'TZERO', + 'TNULL', + 'TDISP', + 'TDMIN', + 'TDMAX', + 'TDESC', + 'TROTA', + 'TRPIX', + 'TRVAL', + 'TDELT', + 'TCUNI', + # 'FZALG' + ] + for i in xrange(1, tfields + 1): + names = ['%s%d' % (n, i) for n in nbase] + self.delete(names) + + def get(self, item, default_value=None): + """ + Get the requested header entry by keyword name + """ + + found, name = self._contains_and_name(item) + if found: + return self._record_map[name]['value'] + else: + return default_value + + def __len__(self): + return len(self._record_list) + + def __contains__(self, item): + found, _ = self._contains_and_name(item) + return found + + def _contains_and_name(self, item): + if isinstance(item, FITSRecord): + name = item['name'] + elif isinstance(item, dict): + name = item.get('name', None) + if name is None: + raise ValueError("dict record must have 'name' field") + else: + name = item + + found = False + if name is None: + if None in self._record_map: + found = True + else: + name = name.upper() + if name in self._record_map: + found = True + elif name[0:8] == 'HIERARCH': + if len(name) > 9: + name = name[9:] + if name in self._record_map: + found = True + + return found, name + + def __setitem__(self, item, value): + if isinstance(value, (dict, FITSRecord)): + if item.upper() != value['name'].upper(): + raise ValueError( + "when setting using a FITSRecord, the " + "name field must match" + ) + rec = value + else: + rec = {'name': item, 'value': value} + + try: + # the entry may already exist; if so, preserve the comment + comment = self.get_comment(item) + rec['comment'] = comment + except KeyError: + pass + + self.add_record(rec) + + def __getitem__(self, item): + if item not in self: + raise KeyError("unknown record: %s" % item) + + return self.get(item) + + def __iter__(self): + self._current = 0 + return self + + def next(self): + """ + for iteration over the header entries + """ + if self._current < len(self._record_list): + rec = self._record_list[self._current] + key = rec['name'] + self._current += 1 + return key + else: + raise StopIteration + + __next__ = next + + def _record2card(self, record): + """ + when we add new records they don't have a card, + this sort of fakes it up similar to what cfitsio + does, just for display purposes. e.g. + + DBL = 23.299843 + LNG = 3423432 + KEYSNC = 'hello ' + KEYSC = 'hello ' / a comment for string + KEYDC = 3.14159265358979 / a comment for pi + KEYLC = 323423432 / a comment for long + + basically, + - 8 chars, left aligned, for the keyword name + - a space + - 20 chars for value, left aligned for strings, right aligned for + numbers + - if there is a comment, one space followed by / then another space + then the comment out to 80 chars + + """ + name = record['name'] + value = record['value'] + comment = record.get('comment', '') + + v_isstring = isstring(value) + + if name is None: + card = ' %s' % comment + elif name == 'COMMENT': + card = 'COMMENT %s' % comment + elif name == 'CONTINUE': + card = 'CONTINUE %s' % value + elif name == 'HISTORY': + card = 'HISTORY %s' % value + else: + if len(name) > 8: + card = 'HIERARCH %s= ' % name + else: + card = '%-8s= ' % name[0:8] + + # these may be string representations of data, or actual strings + if v_isstring: + value = str(value) + if len(value) > 0: + if value[0] != "'": + # this is a string representing a string header field + # make it look like it will look in the header + value = "'" + value + "'" + vstr = '%-20s' % value + else: + vstr = "%20s" % value + else: + vstr = "''" + else: + if value is True: + value = 'T' + elif value is False: + value = 'F' + + # upper for things like 1.0E20 rather than 1.0e20 + vstr = ('%20s' % value).upper() + + card += vstr + + if 'comment' in record: + card += ' / %s' % record['comment'] + + if v_isstring and len(card) > 80: + card = card[0:79] + "'" + else: + card = card[0:80] + + return card + + def __repr__(self): + rep = [''] + for r in self._record_list: + card = self._record2card(r) + # if 'card_string' not in r: + # card = self._record2card(r) + # else: + # card = r['card_string'] + + rep.append(card) + return '\n'.join(rep) + + +class FITSRecord(dict): + """ + Class to represent a FITS header record + + parameters + ---------- + record: string or dict + If a string, it should represent a FITS header card + + If a dict it should have 'name' and 'value' fields. + Can have a 'comment' field. + + examples + -------- + + # from a dict. Can include a comment + rec=FITSRecord( {'name':'temp', 'value':35, 'comment':'temperature in C'} ) + + # from a card + card=FITSRecord('test = 77 / My comment') + + """ + + def __init__(self, record): + self.set_record(record) + + def set_record(self, record, **keys): + """ + check the record is valid and set keys in the dict + + parameters + ---------- + record: string + Dict representing a record or a string representing a FITS header + card + """ + + if keys: + import warnings + + warnings.warn( + "The keyword arguments '%s' are being ignored! This warning " + "will be an error in a future version of `fitsio`!" % keys, + DeprecationWarning, + stacklevel=2, + ) + + if isstring(record): + card = FITSCard(record) + self.update(card) + + self.verify() + + else: + if isinstance(record, FITSRecord): + self.update(record) + elif isinstance(record, dict): + if 'name' in record and 'value' in record: + self.update(record) + + elif 'card_string' in record: + self.set_record(record['card_string']) + + else: + raise ValueError( + 'record must have name,value fields ' + 'or a card_string field' + ) + else: + raise ValueError( + "record must be a string card or dictionary or FITSRecord" + ) + + def verify(self): + """ + make sure name,value exist + """ + if 'name' not in self: + raise ValueError("each record must have a 'name' field") + if 'value' not in self: + raise ValueError("each record must have a 'value' field") + + +_BLANK = ' ' + + +class FITSCard(FITSRecord): + """ + class to represent ordinary FITS cards. + + CONTINUE not supported + + examples + -------- + + # from a card + card=FITSRecord('test = 77 / My comment') + """ + + def __init__(self, card_string): + self.set_card(card_string) + + def set_card(self, card_string): + self['card_string'] = card_string + + self._check_hierarch() + + if self._is_hierarch: + self._set_as_key() + else: + self._check_equals() + + self._check_type() + self._check_len() + + front = card_string[0:7] + if not self.has_equals() or front in [ + 'COMMENT', + 'HISTORY', + 'CONTINU', + _BLANK, + ]: + if front == 'HISTORY': + self._set_as_history() + elif front == 'CONTINU': + self._set_as_continue() + elif front == _BLANK: + self._set_as_blank() + else: + # note anything without an = and not history and not blank + # key comment is treated as COMMENT; this is built into + # cfitsio as well + self._set_as_comment() + + if self.has_equals(): + mess = ( + "warning: It is not FITS-compliant for a %s header " + "card to include an = sign. There may be slight " + "inconsistencies if you write this back out to a " + "file." + ) + mess = mess % (card_string[:8]) + warnings.warn(mess, FITSRuntimeWarning) + else: + self._set_as_key() + + def has_equals(self): + """ + True if = is in position 8 + """ + return self._has_equals + + def _check_hierarch(self): + card_string = self['card_string'] + if card_string[0:8].upper() == 'HIERARCH': + self._is_hierarch = True + else: + self._is_hierarch = False + + def _check_equals(self): + """ + check for = in position 8, set attribute _has_equals + """ + card_string = self['card_string'] + if len(card_string) < 9: + self._has_equals = False + elif card_string[8] == '=': + self._has_equals = True + else: + self._has_equals = False + + def _set_as_key(self): + card_string = self['card_string'] + res = _fitsio_wrap.parse_card(card_string) + if len(res) == 5: + keyclass, name, value, dtype, comment = res + else: + keyclass, name, dtype, comment = res + value = None + + if keyclass == TYP_CONT_KEY: + raise ValueError( + "bad card '%s'. CONTINUE not supported" % card_string + ) + + self['class'] = keyclass + self['name'] = name + self['value_orig'] = value + self['value'] = self._convert_value(value) + self['dtype'] = dtype + self['comment'] = comment + + def _set_as_blank(self): + self['class'] = TYP_USER_KEY + self['name'] = None + self['value'] = None + self['comment'] = self['card_string'][8:] + + def _set_as_comment(self): + comment = self._extract_comm_or_hist_value() + + self['class'] = TYP_COMM_KEY + self['name'] = 'COMMENT' + self['value'] = comment + + def _set_as_history(self): + history = self._extract_comm_or_hist_value() + + self['class'] = TYP_COMM_KEY + self['name'] = 'HISTORY' + self['value'] = history + + def _set_as_continue(self): + value = self._extract_comm_or_hist_value() + + self['class'] = TYP_CONT_KEY + self['name'] = 'CONTINUE' + self['value'] = value + + def _convert_value(self, value_orig): + """ + things like 6 and 1.25 are converted with ast.literal_value + + Things like 'hello' are stripped of quotes + """ + import ast + + if value_orig is None: + return value_orig + + if value_orig.startswith("'") and value_orig.endswith("'"): + value = value_orig[1:-1] + else: + try: + avalue = ast.parse(value_orig).body[0].value + if isinstance(avalue, ast.BinOp): + # this is probably a string that happens to look like + # a binary operation, e.g. '25-3' + value = value_orig + else: + value = ast.literal_eval(value_orig) + except Exception: + value = self._convert_string(value_orig) + + if isinstance(value, int) and '_' in value_orig: + value = value_orig + + return value + + def _convert_string(self, s): + if s == 'T': + return True + elif s == 'F': + return False + else: + return s + + def _extract_comm_or_hist_value(self): + card_string = self['card_string'] + if self._has_equals: + if len(card_string) >= 9: + value = card_string[9:] + else: + value = '' + else: + if len(card_string) >= 8: + # value=card_string[7:] + value = card_string[8:] + else: + value = '' + return value + + def _check_type(self): + card_string = self['card_string'] + if not isstring(card_string): + raise TypeError( + "card must be a string, got type %s" % type(card_string) + ) + + def _check_len(self): + ln = len(self['card_string']) + if ln > 80: + mess = "len(card) is %d. cards must have length < 80" + raise ValueError(mess) diff --git a/fitsio/test_images/test_gzip_compressed_image.fits.fz b/fitsio/test_images/test_gzip_compressed_image.fits.fz new file mode 100644 index 0000000..6dae22d Binary files /dev/null and b/fitsio/test_images/test_gzip_compressed_image.fits.fz differ diff --git a/fitsio/tests/__init__.py b/fitsio/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fitsio/tests/checks.py b/fitsio/tests/checks.py new file mode 100644 index 0000000..3b4a00a --- /dev/null +++ b/fitsio/tests/checks.py @@ -0,0 +1,213 @@ +import sys +import numpy as np +from .. import util + + +def check_header(header, rh): + for k in header: + v = header[k] + rv = rh[k] + + if isinstance(rv, str): + v = v.strip() + rv = rv.strip() + + assert v == rv, "testing equal key '%s'" % k + + +def compare_headerlist_header(header_list, header): + """ + The first is a list of dicts, second a FITSHDR + """ + for entry in header_list: + name = entry['name'].upper() + value = entry['value'] + hvalue = header[name] + + if isinstance(hvalue, str): + hvalue = hvalue.strip() + + assert value == hvalue, "testing header key '%s'" % name + + if 'comment' in entry: + assert ( + entry['comment'].strip() == header.get_comment(name).strip() + ), "testing comment for header key '%s'" % name + + +def cast_shape(shape): + if len(shape) == 2 and shape[1] == 1: + return (shape[0],) + elif shape == (1,): + return tuple() + else: + return shape + + +def compare_array(arr1, arr2, name): + arr1_shape = cast_shape(arr1.shape) + arr2_shape = cast_shape(arr2.shape) + + assert arr1_shape == arr2_shape, ( + "testing arrays '%s' shapes are equal: " + "input %s, read: %s" % (name, arr1_shape, arr2_shape) + ) + + if sys.version_info >= (3, 0, 0) and arr1.dtype.char == 'S': + _arr1 = arr1.astype('U') + else: + _arr1 = arr1 + + res = np.where(_arr1 != arr2) + for i, w in enumerate(res): + assert w.size == 0, "testing array '%s' dim %d are equal" % (name, i) + + +def compare_array_tol(arr1, arr2, tol, name): + assert arr1.shape == arr2.shape, ( + "testing arrays '%s' shapes are equal: " + "input %s, read: %s" % (name, arr1.shape, arr2.shape) + ) + + adiff = np.abs((arr1 - arr2) / arr1) + maxdiff = adiff.max() + res = np.where(adiff > tol) + for i, w in enumerate(res): + assert w.size == 0, ( + "testing array '%s' dim %d are " + "equal within tolerance %e, found " + "max diff %e" % (name, i, tol, maxdiff) + ) + + +def compare_array_abstol(arr1, arr2, tol, name): + assert arr1.shape == arr2.shape, ( + "testing arrays '%s' shapes are equal: " + "input %s, read: %s" % (name, arr1.shape, arr2.shape) + ) + + adiff = np.abs(arr1 - arr2) + maxdiff = adiff.max() + res = np.where(adiff > tol) + for i, w in enumerate(res): + assert w.size == 0, ( + "testing array '%s' dim %d are " + "equal within tolerance %e, found " + "max diff %e" % (name, i, tol, maxdiff) + ) + + +def compare_object_array(arr1, arr2, name, rows=None): + """ + The first must be object + """ + if rows is None: + rows = np.arange(arr1.size) + + for i, row in enumerate(rows): + if ( + sys.version_info >= (3, 0, 0) and isinstance(arr2[i], bytes) + ) or isinstance(arr2[i], str): + if sys.version_info >= (3, 0, 0) and isinstance(arr1[row], bytes): + _arr1row = arr1[row].decode('ascii') + else: + _arr1row = arr1[row] + + assert _arr1row == arr2[i], "%s str el %d equal" % (name, i) + else: + delement = arr2[i] + orig = arr1[row] + s = len(orig) + compare_array( + orig, delement[0:s], "%s num el %d equal" % (name, i) + ) + + +def compare_rec(rec1, rec2, name): + for f in rec1.dtype.names: + rec1_shape = cast_shape(rec1[f].shape) + rec2_shape = cast_shape(rec2[f].shape) + + assert rec1_shape == rec2_shape, ( + "testing '%s' field '%s' shapes are equal: " + "input %s, read: %s" % (name, f, rec1_shape, rec2_shape) + ) + + if sys.version_info >= (3, 0, 0) and rec1[f].dtype.char == 'S': + # for python 3, we get back unicode always + _rec1f = rec1[f].astype('U') + else: + _rec1f = rec1[f] + + assert np.all(_rec1f == rec2[f]) + # res = np.where(_rec1f != rec2[f]) + # for w in res: + # assert w.size == 0, "testing column %s" % f + + +def compare_rec_subrows(rec1, rec2, rows, name): + for f in rec1.dtype.names: + rec1_shape = cast_shape(rec1[f][rows].shape) + rec2_shape = cast_shape(rec2[f].shape) + + assert rec1_shape == rec2_shape, ( + "testing '%s' field '%s' shapes are equal: " + "input %s, read: %s" % (name, f, rec1_shape, rec2_shape) + ) + + if sys.version_info >= (3, 0, 0) and rec1[f].dtype.char == 'S': + # for python 3, we get back unicode always + _rec1frows = rec1[f][rows].astype('U') + else: + _rec1frows = rec1[f][rows] + + res = np.where(_rec1frows != rec2[f]) + for w in res: + assert w.size == 0, "testing column %s" % f + + +def compare_rec_with_var(rec1, rec2, name, rows=None): + """ + + First one *must* be the one with object arrays + + Second can have fixed length + + both should be same number of rows + + """ + + if rows is None: + rows = np.arange(rec2.size) + assert rec1.size == rec2.size, ( + "testing '%s' same number of rows" % name + ) + + # rec2 may have fewer fields + for f in rec2.dtype.names: + # f1 will have the objects + if util.is_object(rec1[f]): + compare_object_array( + rec1[f], + rec2[f], + "testing '%s' field '%s'" % (name, f), + rows=rows, + ) + else: + compare_array( + rec1[f][rows], + rec2[f], + "testing '%s' num field '%s' equal" % (name, f), + ) + + +def compare_names(read_names, true_names, lower=False, upper=False): + for nread, ntrue in zip(read_names, true_names): + if lower: + tname = ntrue.lower() + mess = "lower: '%s' vs '%s'" % (nread, tname) + else: + tname = ntrue.upper() + mess = "upper: '%s' vs '%s'" % (nread, tname) + + assert nread == tname, mess diff --git a/fitsio/tests/makedata.py b/fitsio/tests/makedata.py new file mode 100644 index 0000000..83e0394 --- /dev/null +++ b/fitsio/tests/makedata.py @@ -0,0 +1,439 @@ +import sys +import numpy as np +from functools import lru_cache + +from ..util import cfitsio_version + +CFITSIO_VERSION = cfitsio_version(asfloat=True) + +lorem_ipsum = ( + 'Lorem ipsum dolor sit amet, consectetur adipiscing ' + 'elit, sed do eiusmod tempor incididunt ut labore ' + 'et dolore magna aliqua' +) + + +@lru_cache(maxsize=1) +def make_data(): + nvec = 2 + ashape = (21, 21) + Sdtype = 'S6' + Udtype = 'U6' + + # all currently available types, scalar, 1-d and 2-d array columns + dtype = [ + ('u1scalar', 'u1'), + ('i1scalar', 'i1'), + ('b1scalar', '?'), + ('u2scalar', 'u2'), + ('i2scalar', 'i2'), + ('u4scalar', 'u4'), + ('i4scalar', 'f8'), + ('c8scalar', 'c8'), # complex, two 32-bit + ('c16scalar', 'c16'), # complex, two 32-bit + ('u1vec', 'u1', nvec), + ('i1vec', 'i1', nvec), + ('b1vec', '?', nvec), + ('u2vec', 'u2', nvec), + ('i2vec', 'i2', nvec), + ('u4vec', 'u4', nvec), + ('i4vec', 'i4', nvec), + ('i8vec', 'i8', nvec), + ('f4vec', 'f4', nvec), + ('f8vec', 'f8', nvec), + ('c8vec', 'c8', nvec), + ('c16vec', 'c16', nvec), + ('u1arr', 'u1', ashape), + ('i1arr', 'i1', ashape), + ('b1arr', '?', ashape), + ('u2arr', 'u2', ashape), + ('i2arr', 'i2', ashape), + ('u4arr', 'u4', ashape), + ('i4arr', 'i4', ashape), + ('i8arr', 'i8', ashape), + ('f4arr', 'f4', ashape), + ('f8arr', 'f8', ashape), + ('c8arr', 'c8', ashape), + ('c16arr', 'c16', ashape), + # special case of (1,) + ('f8arr_dim1', 'f8', (1,)), + ('Sscalar', Sdtype), + ('Svec', Sdtype, nvec), + ('Sarr', Sdtype, ashape), + ] + + if CFITSIO_VERSION > 4: + dtype += [ + ('u8scalar', 'u8'), + ('u8vec', 'u8', nvec), + ('u8arr', 'u8', ashape), + ] + + # cfitsio 3 or earlier does not + # handle non-space padded strings + # properly + if CFITSIO_VERSION >= 4: + dtype += [ + ('Sscalar_nopad', Sdtype), + ('Svec_nopad', Sdtype, nvec), + ('Sarr_nopad', Sdtype, ashape), + ] + + if sys.version_info > (3, 0, 0): + dtype += [ + ('Uscalar', Udtype), + ('Uvec', Udtype, nvec), + ('Uarr', Udtype, ashape), + ] + + # cfitsio 3 or earlier does not + # handle non-space padded strings + # properly + if CFITSIO_VERSION >= 4: + dtype += [ + ('Uscalar_nopad', Udtype), + ('Uvec_nopad', Udtype, nvec), + ('Uarr_nopad', Udtype, ashape), + ] + + dtype2 = [ + ('index', 'i4'), + ('x', 'f8'), + ('y', 'f8'), + ] + + nrows = 4 + data = np.zeros(nrows, dtype=dtype) + + dtypes = [ + 'u1', + 'i1', + 'u2', + 'i2', + 'u4', + 'i4', + 'i8', + 'f4', + 'f8', + 'c8', + 'c16', + ] + if CFITSIO_VERSION > 4: + dtypes += ["u8"] + + for t in dtypes: + if t in ['c8', 'c16']: + data[t + 'scalar'] = [ + complex(i + 1, (i + 1) * 2) for i in range(nrows) + ] + vname = t + 'vec' + for row in range(nrows): + for i in range(nvec): + index = (row + 1) * (i + 1) + data[vname][row, i] = complex(index, index * 2) + aname = t + 'arr' + for row in range(nrows): + for i in range(ashape[0]): + for j in range(ashape[1]): + index = (row + 1) * (i + 1) * (j + 1) + data[aname][row, i, j] = complex(index, index * 2) + + else: + data[t + 'scalar'] = 1 + np.arange(nrows, dtype=t) + data[t + 'vec'] = 1 + np.arange( + nrows * nvec, + dtype=t, + ).reshape(nrows, nvec) + arr = 1 + np.arange(nrows * ashape[0] * ashape[1], dtype=t) + data[t + 'arr'] = arr.reshape(nrows, ashape[0], ashape[1]) + + for t in ['b1']: + data[t + 'scalar'] = (np.arange(nrows) % 2 == 0).astype('?') + data[t + 'vec'] = ( + (np.arange(nrows * nvec) % 2 == 0).astype('?').reshape(nrows, nvec) + ) + + arr = (np.arange(nrows * ashape[0] * ashape[1]) % 2 == 0).astype('?') + data[t + 'arr'] = arr.reshape(nrows, ashape[0], ashape[1]) + + # strings get padded when written to the fits file. And the way I do + # the read, I read all bytes (ala mrdfits) so the spaces are preserved. + # + # so we need to pad out the strings with blanks so we can compare + + data['Sscalar'] = ['%-6s' % s for s in ['hello', 'world', 'good', 'bye']] + data['Svec'][:, 0] = '%-6s' % 'hello' + data['Svec'][:, 1] = '%-6s' % 'world' + + s = 1 + np.arange(nrows * ashape[0] * ashape[1]) + s = ['%-6s' % el for el in s] + data['Sarr'] = np.array(s).reshape(nrows, ashape[0], ashape[1]) + + # cfitsio 3 or earlier does not + # handle non-space padded strings + # properly + if CFITSIO_VERSION >= 4: + data['Sscalar_nopad'] = ['hello', 'world', 'good', 'bye'] + data['Svec_nopad'][:, 0] = 'hello' + data['Svec_nopad'][:, 1] = 'world' + + s = 1 + np.arange(nrows * ashape[0] * ashape[1]) + s = ['%s' % el for el in s] + data['Sarr_nopad'] = np.array(s).reshape(nrows, ashape[0], ashape[1]) + + if sys.version_info >= (3, 0, 0): + data['Uscalar'] = [ + '%-6s' % s for s in ['hello', 'world', 'good', 'bye'] + ] + data['Uvec'][:, 0] = '%-6s' % 'hello' + data['Uvec'][:, 1] = '%-6s' % 'world' + + s = 1 + np.arange(nrows * ashape[0] * ashape[1]) + s = ['%-6s' % el for el in s] + data['Uarr'] = np.array(s).reshape(nrows, ashape[0], ashape[1]) + + # cfitsio 3 or earlier does not + # handle non-space padded strings + # properly + if CFITSIO_VERSION >= 4: + data['Uscalar_nopad'] = ['hello', 'world', 'good', 'bye'] + data['Uvec_nopad'][:, 0] = 'hello' + data['Uvec_nopad'][:, 1] = 'world' + + s = 1 + np.arange(nrows * ashape[0] * ashape[1]) + s = ['%s' % el for el in s] + data['Uarr_nopad'] = np.array(s).reshape( + nrows, + ashape[0], + ashape[1], + ) + + # use a dict list so we can have comments + # for long key we used the largest possible + + keys = [ + {'name': 'test1', 'value': 35}, + {'name': 'empty', 'value': ''}, + {'name': 'long_keyword_name', 'value': 'stuff'}, + { + 'name': 'test2', + 'value': 'stuff', + 'comment': 'this is a string keyword', + }, + { + 'name': 'dbl', + 'value': 23.299843, + 'comment': "this is a double keyword", + }, + { + 'name': 'edbl', + 'value': 1.384123233e43, + 'comment': "double keyword with exponent", + }, + { + 'name': 'lng', + 'value': 2**63 - 1, + 'comment': 'this is a long keyword', + }, + {'name': 'lngstr', 'value': lorem_ipsum, 'comment': 'long string'}, + ] + + # a second extension using the convenience function + nrows2 = 10 + data2 = np.zeros(nrows2, dtype=dtype2) + data2['index'] = np.arange(nrows2, dtype='i4') + data2['x'] = np.arange(nrows2, dtype='f8') + data2['y'] = np.arange(nrows2, dtype='f8') + + # + # ascii table + # + + nvec = 2 + ashape = (2, 3) + Sdtype = 'S6' + Udtype = 'U6' + + # we support writing i2, i4, i8, f4 f8, but when reading cfitsio always + # reports their types as i4 and f8, so can't really use i8 and we are + # forced to read all floats as f8 precision + + adtype = [ + ('i2scalar', 'i2'), + ('i4scalar', 'i4'), + # ('i8scalar', 'i8'), + ('f4scalar', 'f4'), + ('f8scalar', 'f8'), + ('Sscalar', Sdtype), + ] + if sys.version_info >= (3, 0, 0): + adtype += [('Uscalar', Udtype)] + + nrows = 4 + try: + tdt = np.dtype(adtype, align=True) + except TypeError: # older numpy may not understand `align` argument + tdt = np.dtype(adtype) + adata = np.zeros(nrows, dtype=tdt) + + adata['i2scalar'][:] = -32222 + np.arange(nrows, dtype='i2') + adata['i4scalar'][:] = -1353423423 + np.arange(nrows, dtype='i4') + adata['f4scalar'][:] = ( + -2.55555555555555555555555e35 + np.arange(nrows, dtype='f4') * 1.0e35 + ) + adata['f8scalar'][:] = ( + -2.55555555555555555555555e110 + np.arange(nrows, dtype='f8') * 1.0e110 + ) + adata['Sscalar'] = ['hello', 'world', 'good', 'bye'] + + if sys.version_info >= (3, 0, 0): + adata['Uscalar'] = ['hello', 'world', 'good', 'bye'] + + ascii_data = adata + + # + # for variable length columns + # + + # all currently available types, scalar, 1-d and 2-d array columns + dtype = [ + ('u1scalar', 'u1'), + ('u1obj', 'O'), + ('i1scalar', 'i1'), + ('i1obj', 'O'), + ('u2scalar', 'u2'), + ('u2obj', 'O'), + ('i2scalar', 'i2'), + ('i2obj', 'O'), + ('u4scalar', 'u4'), + ('u4obj', 'O'), + ('i4scalar', 'f8'), + ('f8obj', 'O'), + ('u1vec', 'u1', nvec), + ('i1vec', 'i1', nvec), + ('u2vec', 'u2', nvec), + ('i2vec', 'i2', nvec), + ('u4vec', 'u4', nvec), + ('i4vec', 'i4', nvec), + ('i8vec', 'i8', nvec), + ('f4vec', 'f4', nvec), + ('f8vec', 'f8', nvec), + ('u1arr', 'u1', ashape), + ('i1arr', 'i1', ashape), + ('u2arr', 'u2', ashape), + ('i2arr', 'i2', ashape), + ('u4arr', 'u4', ashape), + ('i4arr', 'i4', ashape), + ('i8arr', 'i8', ashape), + ('f4arr', 'f4', ashape), + ('f8arr', 'f8', ashape), + # special case of (1,) + ('f8arr_dim1', 'f8', (1,)), + ('Sscalar', Sdtype), + ('Sobj', 'O'), + ('Svec', Sdtype, nvec), + ('Sarr', Sdtype, ashape), + ] + if CFITSIO_VERSION > 4: + dtype += [ + ('u8vec', 'u8', nvec), + ('u8arr', 'u8', ashape), + ('u8scalar', 'i8'), + ('u8obj', 'O'), + ] + + if sys.version_info > (3, 0, 0): + dtype += [ + ('Uscalar', Udtype), + ('Uvec', Udtype, nvec), + ('Uarr', Udtype, ashape), + ] + + nrows = 4 + vardata = np.zeros(nrows, dtype=dtype) + + _dtypes = ['u1', 'i1', 'u2', 'i2', 'u4', 'i4', 'i8', 'f4', 'f8'] + if CFITSIO_VERSION > 4: + _dtypes += ["u8"] + for t in _dtypes: + vardata[t + 'scalar'] = 1 + np.arange(nrows, dtype=t) + vardata[t + 'vec'] = 1 + np.arange(nrows * nvec, dtype=t).reshape( + nrows, + nvec, + ) + arr = 1 + np.arange(nrows * ashape[0] * ashape[1], dtype=t) + vardata[t + 'arr'] = arr.reshape(nrows, ashape[0], ashape[1]) + + for i in range(nrows): + vardata[t + 'obj'][i] = vardata[t + 'vec'][i] + + # strings get padded when written to the fits file. And the way I do + # the read, I real all bytes (ala mrdfits) so the spaces are preserved. + # + # so for comparisons, we need to pad out the strings with blanks so we + # can compare + + vardata['Sscalar'] = [ + '%-6s' % s for s in ['hello', 'world', 'good', 'bye'] + ] + vardata['Svec'][:, 0] = '%-6s' % 'hello' + vardata['Svec'][:, 1] = '%-6s' % 'world' + + s = 1 + np.arange(nrows * ashape[0] * ashape[1]) + s = ['%-6s' % el for el in s] + vardata['Sarr'] = np.array(s).reshape(nrows, ashape[0], ashape[1]) + + if sys.version_info >= (3, 0, 0): + vardata['Uscalar'] = [ + '%-6s' % s for s in ['hello', 'world', 'good', 'bye'] + ] + vardata['Uvec'][:, 0] = '%-6s' % 'hello' + vardata['Uvec'][:, 1] = '%-6s' % 'world' + + s = 1 + np.arange(nrows * ashape[0] * ashape[1]) + s = ['%-6s' % el for el in s] + vardata['Uarr'] = np.array(s).reshape(nrows, ashape[0], ashape[1]) + + for i in range(nrows): + vardata['Sobj'][i] = vardata['Sscalar'][i].rstrip() + + # + # for bitcol columns + # + nvec = 2 + ashape = (21, 21) + + dtype = [('b1vec', '?', nvec), ('b1arr', '?', ashape)] + + nrows = 4 + bdata = np.zeros(nrows, dtype=dtype) + + for t in ['b1']: + bdata[t + 'vec'] = ( + (np.arange(nrows * nvec) % 2 == 0) + .astype('?') + .reshape( + nrows, + nvec, + ) + ) + arr = (np.arange(nrows * ashape[0] * ashape[1]) % 2 == 0).astype('?') + bdata[t + 'arr'] = arr.reshape(nrows, ashape[0], ashape[1]) + + return { + 'data': data, + 'keys': keys, + 'data2': data2, + 'ascii_data': ascii_data, + 'vardata': vardata, + 'bdata': bdata, + } diff --git a/fitsio/tests/test_empty_slice.py b/fitsio/tests/test_empty_slice.py new file mode 100644 index 0000000..3c1a4b7 --- /dev/null +++ b/fitsio/tests/test_empty_slice.py @@ -0,0 +1,20 @@ +import tempfile +import os +import numpy as np +from ..fitslib import write, FITS + + +def test_empty_image_slice(): + shape = (10, 10) + data = np.arange(shape[0] * shape[1]).reshape(shape) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + write(fname, data, clobber=True) + + with FITS(fname) as fits: + assert fits[0][0:0, 0:0].size == 0 + + assert fits[0][0:8, 0:0].size == 0 + + assert fits[0][0:0, 0:8].size == 0 diff --git a/fitsio/tests/test_header.py b/fitsio/tests/test_header.py new file mode 100644 index 0000000..8fd668b --- /dev/null +++ b/fitsio/tests/test_header.py @@ -0,0 +1,576 @@ +import os +import tempfile +import warnings +import numpy as np + +import pytest + +from .makedata import make_data, lorem_ipsum +from .checks import check_header, compare_headerlist_header +from ..fitslib import FITS, read_header, write +from ..header import FITSHDR +from ..hdu.base import INVALID_HDR_CHARS +from ..util import cfitsio_version + +CFITSIO_VERSION = cfitsio_version(asfloat=True) + + +def test_free_form_string(): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with open(fname, 'w') as f: + s = ( + "SIMPLE = T / Standard FITS " # noqa + + "BITPIX = 16 / number of bits per data pixel " # noqa + + "NAXIS = 0 / number of data axes " # noqa + + "EXTEND = T / File contains extensions " # noqa + + "PHOTREF = 'previous MegaCam' / Source: cum.photcat " # noqa + + "EXTRA = 7 / need another line following PHOTREF " # noqa + + "END " # noqa + ) + f.write(s + ' ' * (2880 - len(s))) + hdr = read_header(fname) + assert hdr['PHOTREF'] == 'previous MegaCam' + + +def test_add_delete_and_update_records(): + # Build a FITSHDR from a few records (no need to write on disk) + # Record names have to be in upper case to match with FITSHDR.add_record + recs = [ + {'name': "First_record".upper(), 'value': 1, 'comment': "number 1"}, + {'name': "Second_record".upper(), 'value': "2"}, + {'name': "Third_record".upper(), 'value': "3"}, + {'name': "Last_record".upper(), 'value': 4, 'comment': "number 4"}, + ] + hdr = FITSHDR(recs) + + # Add a new record + hdr.add_record({'name': 'New_record'.upper(), 'value': 5}) + + # Delete number 2 and 4 + hdr.delete('Second_record'.upper()) + hdr.delete('Last_record'.upper()) + + # Update records : first and new one + hdr['First_record'] = 11 + hdr['New_record'] = 3 + + # Do some checks : len and get value/comment + assert len(hdr) == 3 + assert hdr['First_record'] == 11 + assert hdr['New_record'] == 3 + assert hdr['Third_record'] == '3' + assert hdr.get_comment('First_record') == 'number 1' + assert not hdr.get_comment('New_record') + + +def testHeaderCommentPreserved(): + """ + Test that the comment is preserved after resetting the value + """ + + l1 = 'KEY1 = 77 / My comment1' + l2 = 'KEY2 = 88 / My comment2' + hdr = FITSHDR() + hdr.add_record(l1) + hdr.add_record(l2) + + hdr['key1'] = 99 + assert hdr.get_comment('key1') == 'My comment1', 'comment not preserved' + + +def test_header_write_read(): + """ + Test a basic header write and read + + Note the other read/write tests also are checking header writing with + a list of dicts + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.zeros(10) + header = { + 'x': 35, + 'y': 88.215, + 'eval': 1.384123233e43, + 'empty': '', + 'funky': '35-8', # test old bug when strings look + # like expressions + 'name': 'J. Smith', + 'what': '89113e6', # test bug where converted to float + 'und': None, + 'binop': '25-3', # test string with binary operation in it + 'unders': '1_000_000', # test string with underscore + 'longs': lorem_ipsum, + } + if CFITSIO_VERSION > 4.02: + # force hierarch + continue + header["long_keyword_name"] = lorem_ipsum + + fits.write_image(data, header=header) + + rh = fits[0].read_header() + check_header(header, rh) + + with FITS(fname) as fits: + rh = fits[0].read_header() + check_header(header, rh) + + +def test_header_delete(): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.zeros(10) + header1 = {'SCARD': 'one', 'ICARD': 1, 'FCARD': 1.0, 'LCARD': True} + fits.write_image(data, header=header1) + rh = fits[0].read_header() + check_header(header1, rh) + + fits[0].delete_key("SCARD") + del header1["SCARD"] + rh = fits[0].read_header() + check_header(header1, rh) + + fits[0].delete_keys(["ICARD", "FCARD"]) + del header1["ICARD"] + del header1["FCARD"] + rh = fits[0].read_header() + check_header(header1, rh) + + with FITS(fname) as fits: + rh = fits[0].read_header() + check_header(header1, rh) + + +def test_header_update(): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.zeros(10) + header1 = {'SCARD': 'one', 'ICARD': 1, 'FCARD': 1.0, 'LCARD': True} + header2 = { + 'SCARD': 'two', + 'ICARD': 2, + 'FCARD': 2.0, + 'LCARD': False, + 'SNEW': 'two', + 'INEW': 2, + 'FNEW': 2.0, + 'LNEW': False, + } + fits.write_image(data, header=header1) + rh = fits[0].read_header() + check_header(header1, rh) + + # Update header + fits[0].write_keys(header2) + + with FITS(fname) as fits: + rh = fits[0].read_header() + check_header(header2, rh) + + +def test_read_header_case(): + """ + Test read_header with and without case sensitivity + + The reason we need a special test for this is because + the read_header code is optimized for speed and has + a different code path + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.zeros(10) + adata = make_data() + fits.write_image(data, header=adata['keys'], extname='First') + fits.write_image(data, header=adata['keys'], extname='second') + + cases = [ + ('First', True), + ('FIRST', False), + ('second', True), + ('seConD', False), + ] + for ext, ci in cases: + h = read_header(fname, ext=ext, case_sensitive=ci) + compare_headerlist_header(adata['keys'], h) + + +def test_blank_key_comments(): + """ + test a few different comments + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + records = [ + # empty should return empty + {'name': None, 'value': '', 'comment': ''}, + # this will also return empty + {'name': None, 'value': '', 'comment': ' '}, + # this will return exactly + {'name': None, 'value': '', 'comment': ' h'}, + # this will return exactly + {'name': None, 'value': '', 'comment': '--- test comment ---'}, + ] + header = FITSHDR(records) + + fits.write(None, header=header) + + rh = fits[0].read_header() + + rrecords = rh.records() + + for i, ri in ((0, 6), (1, 7), (2, 8)): + rec = records[i] + rrec = rrecords[ri] + + assert rec['name'] is None, 'checking name is None' + + comment = rec['comment'] + rcomment = rrec['comment'] + if '' == comment.strip(): + comment = '' + + assert comment == rcomment, "check empty key comment" + + +def test_blank_key_comments_from_cards(): + """ + test a few different comments + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + records = [ + ' ', # noqa + ' --- testing comment --- ', # noqa + ' --- testing comment --- ', # noqa + "COMMENT testing ", # noqa + ] + header = FITSHDR(records) + + fits.write(None, header=header) + + rh = fits[0].read_header() + + rrecords = rh.records() + + assert rrecords[6]['name'] is None, 'checking name is None' + assert rrecords[6]['comment'] == '', 'check empty key comment' + assert rrecords[7]['name'] is None, 'checking name is None' + assert rrecords[7]['comment'] == ' --- testing comment ---', ( + "check empty key comment" + ) + assert rrecords[8]['name'] is None, 'checking name is None' + assert rrecords[8]['comment'] == '--- testing comment ---', ( + "check empty key comment" + ) + assert rrecords[9]['name'] == 'COMMENT', 'checking name is COMMENT' + assert rrecords[9]['comment'] == 'testing', "check comment" + + +def test_header_from_cards(): + """ + test generating a header from cards, writing it out and getting + back what we put in + """ + hdr_from_cards = FITSHDR( + [ + "IVAL = 35 / integer value ", # noqa + "SHORTS = 'hello world' ", # noqa + "UND = ", # noqa + "LONGS = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiu&'", # noqa + "CONTINUE 'smod tempor incididunt ut labore et dolore magna aliqua' ", # noqa + "DBL = 1.25 ", # noqa + ] + ) + header = [ + {'name': 'ival', 'value': 35, 'comment': 'integer value'}, + {'name': 'shorts', 'value': 'hello world'}, + {'name': 'und', 'value': None}, + {'name': 'longs', 'value': lorem_ipsum}, + {'name': 'dbl', 'value': 1.25}, + ] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.zeros(10) + fits.write_image(data, header=hdr_from_cards) + + rh = fits[0].read_header() + compare_headerlist_header(header, rh) + + with FITS(fname) as fits: + rh = fits[0].read_header() + compare_headerlist_header(header, rh) + + +def test_bad_header_write_raises(): + """ + Test that an invalid header raises. + """ + + for c in INVALID_HDR_CHARS: + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + try: + hdr = {'bla%sg' % c: 3} + data = np.zeros(10) + + write(fname, data, header=hdr, clobber=True) + except Exception as e: + assert "header key 'BLA%sG' has" % c in str(e) + + +def test_header_template(): + """ + test adding bunch of cards from a split template + """ + + header_template = """SIMPLE = T / +BITPIX = 8 / bits per data value +NAXIS = 0 / number of axes +EXTEND = T / Extensions are permitted +ORIGIN = 'LSST DM Header Service'/ FITS file originator + + ---- Date, night and basic image information ---- +DATE = / Creation Date and Time of File +DATE-OBS= / Date of the observation (image acquisition) +DATE-BEG= / Time at the start of integration +DATE-END= / end date of the observation +MJD = / Modified Julian Date that the file was written +MJD-OBS = / Modified Julian Date of observation +MJD-BEG = / Modified Julian Date derived from DATE-BEG +MJD-END = / Modified Julian Date derived from DATE-END +OBSID = / ImageName from Camera StartIntergration +GROUPID = / imageSequenceName from StartIntergration +OBSTYPE = / BIAS, DARK, FLAT, OBJECT +BUNIT = 'adu ' / Brightness units for pixel array + + ---- Telescope info, location, observer ---- +TELESCOP= 'LSST AuxTelescope' / Telescope name +INSTRUME= 'LATISS' / Instrument used to obtain these data +OBSERVER= 'LSST' / Observer name(s) +OBS-LONG= -70.749417 / [deg] Observatory east longitude +OBS-LAT = -30.244639 / [deg] Observatory latitude +OBS-ELEV= 2663.0 / [m] Observatory elevation +OBSGEO-X= 1818938.94 / [m] X-axis Geocentric coordinate +OBSGEO-Y= -5208470.95 / [m] Y-axis Geocentric coordinate +OBSGEO-Z= -3195172.08 / [m] Z-axis Geocentric coordinate + + ---- Pointing info, etc. ---- + +DECTEL = / Telescope DEC of observation +ROTPATEL= / Telescope Rotation +ROTCOORD= 'sky' / Telescope Rotation Coordinates +RA = / RA of Target +DEC = / DEC of Target +ROTPA = / Rotation angle relative to the sky (deg) +HASTART = / [HH:MM:SS] Telescope hour angle at start +ELSTART = / [deg] Telescope zenith distance at start +AZSTART = / [deg] Telescope azimuth angle at start +AMSTART = / Airmass at start +HAEND = / [HH:MM:SS] Telescope hour angle at end +ELEND = / [deg] Telescope zenith distance at end +AZEND = / [deg] Telescope azimuth angle at end +AMEND = / Airmass at end + + ---- Image-identifying used to build OBS-ID ---- +TELCODE = 'AT' / The code for the telecope +CONTRLLR= / The controller (e.g. O for OCS, C for CCS) +DAYOBS = / The observation day as defined by image name +SEQNUM = / The sequence number from the image name +GROUPID = / + + ---- Information from Camera +CCD_MANU= 'ITL' / CCD Manufacturer +CCD_TYPE= '3800C' / CCD Model Number +CCD_SERN= '20304' / Manufacturers? CCD Serial Number +LSST_NUM= 'ITL-3800C-098' / LSST Assigned CCD Number +SEQCKSUM= / Checksum of Sequencer +SEQNAME = / SequenceName from Camera StartIntergration +REBNAME = / Name of the REB +CONTNUM = / CCD Controller (WREB) Serial Number +IMAGETAG= / DAQ Image id +TEMP_SET= / Temperature set point (deg C) +CCDTEMP = / Measured temperature (deg C) + + ---- Geometry from Camera ---- +DETSIZE = / Size of sensor +OVERH = / Over-scan pixels +OVERV = / Vert-overscan pix +PREH = / Pre-scan pixels + + ---- Filter/grating information ---- +FILTER = / Name of the filter +FILTPOS = / Filter position +GRATING = / Name of the second disperser +GRATPOS = / disperser position +LINSPOS = / Linear Stage + + ---- Exposure-related information ---- +EXPTIME = / Exposure time in seconds +SHUTTIME= / Shutter exposure time in seconds +DARKTIME= / Dark time in seconds + + ---- Header information ---- +FILENAME= / Original file name +HEADVER = / Version of header + + ---- Checksums ---- +CHECKSUM= / checksum for the current HDU +DATASUM = / checksum of the data records\n""" + + lines = header_template.splitlines() + hdr = FITSHDR() + for line in lines: + hdr.add_record(line) + + +def test_corrupt_continue(): + """ + test with corrupt continue, just make sure it doesn't crash + """ + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with warnings.catch_warnings(record=True) as _: + hdr_from_cards = FITSHDR( + [ + "IVAL = 35 / integer value ", # noqa + "SHORTS = 'hello world' ", # noqa + "CONTINUE= ' ' / '&' / Current observing orogram ", # noqa + "UND = ", # noqa + "DBL = 1.25 ", # noqa + ] + ) + + with FITS(fname, 'rw') as fits: + fits.write(None, header=hdr_from_cards) + + read_header(fname) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with warnings.catch_warnings(record=True) as _: + hdr_from_cards = FITSHDR( + [ + "IVAL = 35 / integer value ", # noqa + "SHORTS = 'hello world' ", # noqa + "PROGRAM = 'Setting the Scale: Determining the Absolute Mass Normalization and &'", # noqa + "CONTINUE 'Scaling Relations for Clusters at z~0.1&' ", # noqa + "CONTINUE '&' / Current observing orogram ", # noqa + "UND = ", # noqa + "DBL = 1.25 ", # noqa + ] + ) + + with FITS(fname, 'rw') as fits: + fits.write(None, header=hdr_from_cards) + + read_header(fname) + + +def record_exists(header_records, key, value): + for rec in header_records: + if rec['name'] == key and rec['value'] == value: + return True + + return False + + +def test_read_comment_history(): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.arange(100).reshape(10, 10) + fits.create_image_hdu(data) + hdu = fits[-1] + hdu.write_comment('A COMMENT 1') + hdu.write_comment('A COMMENT 2') + hdu.write_history('SOME HISTORY 1') + hdu.write_history('SOME HISTORY 2') + fits.close() + + with FITS(fname, 'r') as fits: + hdu = fits[-1] + header = hdu.read_header() + records = header.records() + assert record_exists(records, 'COMMENT', 'A COMMENT 1') + assert record_exists(records, 'COMMENT', 'A COMMENT 2') + assert record_exists(records, 'HISTORY', 'SOME HISTORY 1') + assert record_exists(records, 'HISTORY', 'SOME HISTORY 2') + + +def test_write_key_dict(): + """ + test that write_key works using a standard key dict + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with FITS(fname, 'rw') as fits: + im = np.zeros((10, 10), dtype='i2') + fits.write(im) + + keydict = { + 'name': 'test', + 'value': 35, + 'comment': 'keydict test', + } + fits[-1].write_key(**keydict) + + h = fits[-1].read_header() + + assert h['test'] == keydict['value'] + assert h.get_comment('test') == keydict['comment'] + + +@pytest.mark.parametrize("fname", ["test.fits", "mem://"]) +def test_header_update_compressed_image_to_table(fname): + data = np.arange(10).reshape(5, 2).astype(np.float32) + + fname = "test.fits" + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(data, compress="RICE", qlevel=1, dither_seed=10) + hdr = fits[1].read_header() + + info_before = fits[1].get_info() + for key in hdr.keys(): + if key.startswith("Z"): + fits[1].delete_key(key) + for i in range(1000): + fits[1].write_key("test" + str(i), "blah") + fits[1].delete_key("test0") + + fits.update_hdu_list() + info_after = fits[1].get_info() + + assert info_after != info_before + assert fits[1].get_exttype() == "BINARY_TBL" + + +if __name__ == '__main__': + test_header_write_read() diff --git a/fitsio/tests/test_header_junk.py b/fitsio/tests/test_header_junk.py new file mode 100644 index 0000000..8daa480 --- /dev/null +++ b/fitsio/tests/test_header_junk.py @@ -0,0 +1,66 @@ +import os +import tempfile +import pytest +from ..fitslib import read_header, FITS +from ..fits_exceptions import FITSFormatError + + +def test_header_junk(): + """ + test lenient treatment of garbage written by IDL mwrfits + """ + + data = b"""SIMPLE = T /Primary Header created by MWRFITS v1.11 BITPIX = 16 / NAXIS = 0 / EXTEND = T /Extensions may be present BLAT = 1 /integer FOO = 1.00000 /float (or double?) BAR@ = NAN /float NaN BI.Z = NaN /double NaN BAT = INF /1.0 / 0.0 BOO = -INF /-1.0 / 0.0 QUAT = ' ' /blank string QUIP = '1.0 ' /number in quotes QUIZ = ' 1.0 ' /number in quotes with a leading space QUI\xf4\x04 = 'NaN ' /NaN in quotes HIERARCH QU.@D = 'Inf ' END """ # noqa + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with open(fname, 'wb') as fobj: + fobj.write(data) + + h = read_header(fname) + # these keys are not hierarch but we can parse the name and then + # leave the value as a string, so we do that. + assert h['bar@'] == 'NAN', 'NAN garbage' + assert h['bi.z'] == 'NaN', 'NaN garbage' + assert h['bat'] == 'INF', 'INF garbage' + assert h['boo'] == '-INF', '-INF garbage' + assert h['quat'] == '', 'blank' + assert h['quip'] == '1.0', '1.0 in quotes' + assert h['quiz'] == ' 1.0', '1.0 in quotes' + # the key in the header is 'QUI' + two non-ascii chars and gets + # translated to `QUI__` + assert h['qui__'] == 'NaN', 'NaN in quotes' + # this key is `HIERARCH QU.@D` in the header and so gets read as is + assert h['qu.@d'] == 'Inf', 'Inf in quotes' + + +def test_Header_Junk_Non_Ascii(): + data = b"SIMPLE = T / file does conform to FITS standard BITPIX = 16 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H @\x0f@\x0f \x02\x05\x18@\x02\x02\xc5@\x0c\x03\xf3@\x080\x02\x03\xbc@\x0f@@@@@@@@ END " # noqa + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with open(fname, 'wb') as fobj: + fobj.write(data) + + h = read_header(fname) + assert h["@_@_"] is None + + +def test_missing_xtension_keyword(): + """ + Misformatted header with extension not properly marked with + XTENSION + """ + + data = b"""SIMPLE = T / This is a FITS file BITPIX = 8 / NAXIS = 0 / EXTEND = T / This file may contain FITS extensions NEXTEND = 7 / Number of extensions END SIMPLE = T / file does conform to FITS standard BITPIX = 32 / number of bits per data pixel NAXIS = 2 / number of data axes NAXIS1 = 30 / length of data axis 1 NAXIS2 = 30 / length of data axis 2 EXTEND = T / FITS dataset may contain extensions END """ # noqa + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with open(fname, 'wb') as fobj: + fobj.write(data) + + with pytest.raises(FITSFormatError): + with FITS(fname) as fits: + print(fits) diff --git a/fitsio/tests/test_image.py b/fitsio/tests/test_image.py new file mode 100644 index 0000000..ab2e65e --- /dev/null +++ b/fitsio/tests/test_image.py @@ -0,0 +1,804 @@ +import os +import tempfile + +import pytest + +# import warnings +from .checks import check_header, compare_array +from ..util import cfitsio_version, cfitsio_is_bundled +import numpy as np +from ..fitslib import FITS + +CFITSIO_VERSION = cfitsio_version(asfloat=True) +DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] +if CFITSIO_VERSION > 3.44: + DTYPES += ["u8"] + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_write_read(with_nan): + """ + Test a basic image write, data and a header, then reading back in to + check the values + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with FITS(fname, 'rw') as fits: + # note mixing up byte orders a bit + for dtype in DTYPES: + data = np.arange(5 * 20, dtype=dtype).reshape(5, 20) + if "f" in dtype and with_nan: + data[3, 13] = np.nan + + header = {'DTYPE': dtype, 'NBYTES': data.dtype.itemsize} + fits.write_image(data, header=header) + rdata = fits[-1].read() + + np.testing.assert_array_equal(data, rdata) + + rh = fits[-1].read_header() + check_header(header, rh) + + with FITS(fname) as fits: + for i in range(len(DTYPES)): + assert not fits[i].is_compressed(), 'not compressed' + + +@pytest.mark.parametrize("fname", ["mem://", "test.fits"]) +def test_image_write_read_bool(fname): + rng = np.random.RandomState(seed=10) + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + a = rng.rand(10) + fits.write(a) + a = rng.rand(10) > 0.5 + with pytest.raises(TypeError) as e: + fits.write(a) + + assert "Unsupported numpy image datatype 0" in str(e) + + +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize("dtype", DTYPES) +def test_image_write_read_unaligned(dtype, with_nan): + """ + Test a basic image write, data and a header, then reading back in to + check the values. The data from numpy is an unaligned view. The code + to make the unaligned view was generated by Google's AI and then modified + by hand to fix a bug. + """ + + if ( + dtype == ">f4" or ("f" in dtype and with_nan) + ) and not cfitsio_is_bundled(): + pytest.xfail( + reason=( + "Non-bundled cfitsio libraries have a bug for " + "underflow handling. " + "See https://github.com/HEASARC/cfitsio/pull/102." + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with FITS(fname, 'rw') as fits: + # note mixing up byte orders a bit + data = np.arange(20, dtype=dtype) + unaligned_data = np.ndarray( + shape=(19,), + dtype=data.dtype, + buffer=data.data, + offset=1, # Offset by 1 byte + strides=data.strides, + ) + if not dtype.endswith("1"): + assert not unaligned_data.flags["ALIGNED"] + + if "f" in dtype and with_nan: + unaligned_data[3] = np.nan + + header = { + 'DTYPE': dtype, + 'NBYTES': unaligned_data.dtype.itemsize, + } + fits.write_image(unaligned_data, header=header) + rdata = fits[-1].read() + + np.testing.assert_array_equal(unaligned_data, rdata) + + rh = fits[-1].read_header() + check_header(header, rh) + + with FITS(fname) as fits: + assert not fits[0].is_compressed(), 'not compressed' + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_subnormal_float32(with_nan): + if not cfitsio_is_bundled(): + pytest.xfail( + reason=( + "Non-bundled cfitsio libraries have a bug for " + "underflow handling. " + "See https://github.com/HEASARC/cfitsio/pull/102." + ), + ) + v = 8.82818e-44 + v = [v] * 10 + if with_nan: + v += [np.nan] + nv = np.array(v, dtype=np.float32) + + with FITS("mem://", 'rw') as fits: + fits.write_image(nv) + rdata = fits[-1].read() + + np.testing.assert_array_equal(rdata, nv) + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_subnormal_float64(with_nan): + if not cfitsio_is_bundled(): + pytest.xfail( + reason=( + "Non-bundled cfitsio libraries have a bug for " + "underflow handling. " + "See https://github.com/HEASARC/cfitsio/pull/102." + ), + ) + v = 2.225073858507203e-309 + v = [v] * 10 + if with_nan: + v += [np.nan] + nv = np.array(v, dtype=np.float64) + + with FITS("mem://", 'rw') as fits: + fits.write_image(nv) + rdata = fits[-1].read() + + np.testing.assert_array_equal(rdata, nv) + + +def test_image_write_empty(): + """ + Test a basic image write, with no data and just a header, then reading + back in to check the values + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + data = None + + header = { + 'EXPTIME': 120, + 'OBSERVER': 'Beatrice Tinsley', + 'INSTRUME': 'DECam', + 'FILTER': 'r', + } + ccds = ['CCD1', 'CCD2', 'CCD3', 'CCD4', 'CCD5', 'CCD6', 'CCD7', 'CCD8'] + with FITS(fname, 'rw', ignore_empty=True) as fits: + for extname in ccds: + fits.write_image(data, header=header) + _ = fits[-1].read() + rh = fits[-1].read_header() + check_header(header, rh) + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_write_read_from_dims(with_nan): + """ + Test creating an image from dims and writing in place + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # note mixing up byte orders a bit + for dtype in DTYPES: + data = np.arange(5 * 20, dtype=dtype).reshape(5, 20) + if "f" in dtype and with_nan: + data[3, 13] = np.nan + + fits.create_image_hdu(dims=data.shape, dtype=data.dtype) + + fits[-1].write(data) + rdata = fits[-1].read() + + np.testing.assert_array_equal(data, rdata) + + with FITS(fname) as fits: + for i in range(len(DTYPES)): + assert not fits[i].is_compressed(), "not compressed" + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_write_read_from_dims_chunks(with_nan): + """ + Test creating an image and reading/writing chunks + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # note mixing up byte orders a bit + for dtype in DTYPES: + data = np.arange(5 * 3, dtype=dtype).reshape(5, 3) + if "f" in dtype and with_nan: + data[3, 1] = np.nan + + fits.create_image_hdu(dims=data.shape, dtype=data.dtype) + + chunk1 = data[0:2, :] + chunk2 = data[2:, :] + + # + # first using scalar pixel offset + # + + fits[-1].write(chunk1) + + start = chunk1.size + fits[-1].write(chunk2, start=start) + + rdata = fits[-1].read() + + np.testing.assert_array_equal(data, rdata) + + # + # now using sequence, easier to calculate + # + + fits.create_image_hdu(dims=data.shape, dtype=data.dtype) + + # first using pixel offset + fits[-1].write(chunk1) + + start = [2, 0] + fits[-1].write(chunk2, start=start) + + rdata2 = fits[-1].read() + + np.testing.assert_array_equal(data, rdata2) + + with FITS(fname) as fits: + for i in range(len(DTYPES)): + assert not fits[i].is_compressed(), "not compressed" + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_slice(with_nan): + """ + test reading an image slice + """ + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # note mixing up byte orders a bit + for dtype in DTYPES: + data = np.arange(16 * 20, dtype=dtype).reshape(16, 20) + if "f" in dtype and with_nan: + data[3, 13] = np.nan + + header = {'DTYPE': dtype, 'NBYTES': data.dtype.itemsize} + + fits.write_image(data, header=header) + rdata = fits[-1][4:12, 9:17] + + np.testing.assert_array_equal(data[4:12, 9:17], rdata) + + rh = fits[-1].read_header() + check_header(header, rh) + + +def _check_shape(expected_data, rdata): + mess = 'Data are not the same (Expected shape: %s, actual shape: %s.' % ( + expected_data.shape, + rdata.shape, + ) + np.testing.assert_array_equal(expected_data, rdata, mess) + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_read_flip_axis_slice(with_nan): + """ + Test reading a slice when the slice's start is less than the slice's stop. + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + dtype = np.float32 + data = np.arange(100 * 200, dtype=dtype).reshape(100, 200) + if with_nan: + data[3, 13] = np.nan + fits.write_image(data) + hdu = fits[-1] + rdata = hdu[:, 130:70] + + # Expanded by two to emulate adding one to the start value, and + # adding one to the calculated dimension. + expected_data = data[:, 130:70:-1] + + _check_shape(expected_data, rdata) + + rdata = hdu[:, 130:70:-6] + expected_data = data[:, 130:70:-6] + _check_shape(expected_data, rdata) + + # Expanded by two to emulate adding one to the start value, and + # adding one to the calculated dimension. + expected_data = data[:, 130:70:-6] + _check_shape(expected_data, rdata) + + # Positive step integer with start > stop will return an empty + # array + rdata = hdu[:, 90:60:4] + expected_data = np.empty(0, dtype=dtype) + _check_shape(expected_data, rdata) + + # Negative step integer with start < stop will return an empty + # array. + rdata = hdu[:, 60:90:-4] + expected_data = np.empty(0, dtype=dtype) + _check_shape(expected_data, rdata) + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_image_slice_striding(with_nan): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # note mixing up byte orders a bit + for dtype in DTYPES: + data = np.arange(16 * 20, dtype=dtype).reshape(16, 20) + if "f" in dtype and with_nan: + data[3, 13] = np.nan + header = {'DTYPE': dtype, 'NBYTES': data.dtype.itemsize} + fits.write_image(data, header=header) + + rdata = fits[-1][4:16:4, 2:20:2] + expected_data = data[4:16:4, 2:20:2] + assert rdata.shape == expected_data.shape, ( + "Shapes differ with dtype %s" % dtype + ) + np.testing.assert_array_equal( + expected_data, rdata, "images with dtype %s" % dtype + ) + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_read_ignore_scaling(with_nan): + """ + Test the flag to ignore scaling when reading an HDU. + """ + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + dtype = 'i2' + data = np.arange(10 * 20, dtype=dtype).reshape(10, 20) + if "f" in dtype and with_nan: + data[3, 13] = np.nan + header = { + 'DTYPE': dtype, + 'BITPIX': 16, + 'NBYTES': data.dtype.itemsize, + 'BZERO': 9.33, + 'BSCALE': 3.281, + } + + fits.write_image(data, header=header) + hdu = fits[-1] + + rdata = hdu.read() + assert rdata.dtype == np.float32, 'Wrong dtype.' + + hdu.ignore_scaling = True + rdata = hdu[:, :] + assert rdata.dtype == dtype, 'Wrong dtype when ignoring.' + np.testing.assert_array_equal( + data, rdata, err_msg='Wrong unscaled data.' + ) + + rh = fits[-1].read_header() + check_header(header, rh) + + hdu.ignore_scaling = False + rdata = hdu[:, :] + assert rdata.dtype == np.float32, 'Wrong dtype when not ignoring.' + np.testing.assert_array_equal( + data.astype(np.float32), + rdata, + err_msg='Wrong scaled data returned.', + ) + + +@pytest.mark.parametrize( + "compress_kws", + [ + {}, + { + "compress": "RICE", + "tile_dims": (3, 1, 2), + "qlevel": 2048, + "dither_seed": 10, + }, + { + "compress": "GZIP", + "tile_dims": (3, 1, 2), + "qlevel": 0, + "dither_seed": 10, + }, + ], +) +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize("fname", ["mem://", "test.fits"]) +@pytest.mark.parametrize("sx", [0, 6, 9]) +@pytest.mark.parametrize("sy", [0, 3, 4]) +@pytest.mark.parametrize("sz", [0, 2, 5]) +def test_image_write_subset_3d(sx, sy, sz, fname, with_nan, compress_kws): + rng = np.random.RandomState(seed=10) + img = np.arange(300).reshape(6, 5, 10).astype(np.float32) + img2 = (rng.normal(size=30).reshape(3, 2, 5) * 1000).astype(np.float32) + if with_nan: + img2[0, 1, 2] = np.nan + + if compress_kws and (sx > 5 or sy > 3 or sz > 3): + pytest.skip(reason="tile-compressed fits images cannot be resized!") + + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(img, **compress_kws) + if compress_kws: + fits[1].write(img2, start=[sz, sy, sx]) + img_final = fits[1].read() + else: + fits[0].write(img2, start=[sz, sy, sx]) + img_final = fits[0].read() + + if ( + "compress" in compress_kws + and compress_kws.get("qlevel", np.inf) != 0 + ): + np.testing.assert_allclose( + img_final[ + sz : sz + img2.shape[0], + sy : sy + img2.shape[1], + sx : sx + img2.shape[2], + ], + img2, + ) + else: + np.testing.assert_array_equal( + img_final[ + sz : sz + img2.shape[0], + sy : sy + img2.shape[1], + sx : sx + img2.shape[2], + ], + img2, + ) + + +@pytest.mark.parametrize( + "compress_kws", + [ + {}, + { + "compress": "RICE", + "tile_dims": (5, 2), + "qlevel": 128, + "dither_seed": 10, + }, + {"compress": "GZIP", "tile_dims": (5, 2), "qlevel": 0}, + ], +) +@pytest.mark.parametrize("with_nan_base_img", [False, True]) +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize( + "fname", + [ + "mem://", + "test.fits", + ], +) +@pytest.mark.parametrize("sx", [0, 1, 9]) +@pytest.mark.parametrize("sy", [0, 1, 9]) +@pytest.mark.parametrize("xnan", [0, 1, 9]) +@pytest.mark.parametrize("ynan", [0, 1, 9]) +def test_image_write_subset_2d( + sx, sy, fname, with_nan, compress_kws, with_nan_base_img, xnan, ynan +): + rng = np.random.RandomState(seed=10) + img = np.arange(100).reshape(10, 10) + nse = rng.normal(size=100).reshape(10, 10) + img = (img + 1e-4 * nse).reshape(10, 10).astype(np.float32) + img2 = (10 + rng.normal(size=6).reshape(3, 2)).astype(np.float32) + if with_nan_base_img: + img[ynan, xnan] = np.nan + if with_nan: + img2[1, 0] = np.nan + + if compress_kws and (sx > 8 or sy > 7): + pytest.skip(reason="tile-compressed fits images cannot be resized!") + + if compress_kws and (sx == 9 or sy == 9): + pytest.skip(reason="tile-compressed fits images cannot be resized!") + + # these test cases have the subset image img2 overlapping two + # different compressed image tiles which causes a bug when + # combined with one of the tiles changing its compression type + # due to an edge case in the compression algorithm + partial_overlap_str = f"{xnan}-{ynan}-{sx}-{sy}" + partial_overlap_str_cases = [ + "0-0-0-1", + "0-0-1-0", + "0-0-1-1", + "0-1-1-0", + "0-1-1-1", + "1-0-0-1", + "1-0-1-1", + ] + if ( + with_nan + and with_nan_base_img + and partial_overlap_str in partial_overlap_str_cases + and not cfitsio_is_bundled() + and compress_kws + and compress_kws.get("qlevel", 0) > 0 + ): + pytest.xfail( + reason=( + "Non-bundled cfitsio libraries have a bug for " + "overwriting tile-compressed images in an edge case. " + "See https://github.com/HEASARC/cfitsio/pull/101." + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(img, **compress_kws) + + if compress_kws: + img_final = fits[1].read() + else: + img_final = fits[0].read() + + np.testing.assert_allclose( + img, + img_final, + atol=1e-3, + rtol=0.2, + ) + + if compress_kws: + fits[1].write(img2, start=[sy, sx]) + else: + fits[0].write(img2, start=[sy, sx]) + + if compress_kws: + img_final = fits[1].read() + else: + img_final = fits[0].read() + + if compress_kws: + img_final_slice = fits[1][ + sy : sy + img2.shape[0], sx : sx + img2.shape[1] + ] + else: + img_final_slice = fits[0][ + sy : sy + img2.shape[0], sx : sx + img2.shape[1] + ] + + if ( + "compress" in compress_kws + and compress_kws.get("qlevel", np.inf) != 0 + ): + np.testing.assert_allclose( + img_final[sy : sy + img2.shape[0], sx : sx + img2.shape[1]], + img2, + atol=0, + rtol=0.2, + ) + else: + np.testing.assert_array_equal( + img_final[sy : sy + img2.shape[0], sx : sx + img2.shape[1]], + img2, + ) + + np.testing.assert_array_equal( + img_final[sy : sy + img2.shape[0], sx : sx + img2.shape[1]], + img_final_slice, + ) + + +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize("fname", ["mem://", "test.fits"]) +@pytest.mark.parametrize("sx", [0, 13, 99]) +def test_image_write_subset_1d(sx, fname, with_nan): + rng = np.random.RandomState(seed=10) + img = np.arange(100) + img2 = (rng.normal(size=6) * 1000).astype(np.int_) + if with_nan: + img = img.astype(np.float32) + img2 = img2.astype(np.float32) + img2[5] = np.nan + + for _sx in [sx, [sx]]: + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(img) + fits[0].write(img2, start=_sx) + img_final = fits[0].read() + + np.testing.assert_array_equal( + img_final[sx : sx + img2.shape[0]], + img2, + ) + + +@pytest.mark.parametrize("fname", ["mem://", "test.fits"]) +@pytest.mark.parametrize( + "shape,reshape", + [ + ((6, 5, 10), (10, 12, 23)), + ((1,), (10,)), + ((6, 5), (10, 12)), + ((6, 5, 10), (3, 2, 7)), + ((10,), (3,)), + ((10, 5), (12, 2)), + ], +) +def test_image_reshape(shape, reshape, fname): + img = np.arange(int(np.prod(shape))).reshape(shape) + + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(img) + fits[0].reshape(reshape) + img_final = fits[0].read() + + nel = img.ravel().shape[0] + nel_final = img_final.ravel().shape[0] + min_nel = min(nel, nel_final) + assert np.array_equal( + img_final.ravel()[:min_nel], + img.ravel()[:min_nel], + ) + if nel_final > nel: + assert np.array_equal( + img_final.ravel()[nel:], + np.zeros(nel_final - nel), + ) + + +@pytest.mark.parametrize("fname", ["mem://", "test.fits"]) +@pytest.mark.parametrize( + "dims", + [ + ((1,)), + ( + ( + 2, + 3, + ) + ), + ( + 4, + 5, + 6, + ), + ], +) +def test_image_write_subset_raises(dims, fname): + ndims = len(dims) + rng = np.random.RandomState(seed=10) + img = np.arange(int(np.prod(dims))).reshape(dims) + exdims = dims + (5,) + img2 = ( + rng.normal(size=int(np.prod(exdims))).reshape(exdims) * 1000 + ).astype(np.int_) + + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(img) + with pytest.raises(ValueError) as err: + fits[0].write(img2, start=0) + assert ( + "the input image must have the same number of dimensions" + in str(err.value) + ) + + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write(img) + if ndims > 1: + with pytest.raises(ValueError) as err: + fits[0].write(img2[..., 0], start=9999) + assert ( + "the start keyword must have the same number of dimensions" + in str(err.value) + ) + else: + fits[0].write(img2[..., 0], start=9999) + + with tempfile.TemporaryDirectory() as tmpdir: + if "fpth" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS("mem://", "rw") as fits: + fits.write(img) + if ndims > 1: + fits[0].write(img2[..., :-1, 0], start=1) + else: + fits[0].write(img2[..., 0], start=1) + + +def test_image_read_write_ulonglong(): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with FITS(fname, 'rw') as fits: + data = np.arange(5 * 20, dtype='u8').reshape(5, 20) + header = {'DTYPE': 'u8', 'NBYTES': data.dtype.itemsize} + if CFITSIO_VERSION < 3.45: + with pytest.raises(TypeError) as e: + fits.write_image(data, header=header) + assert ( + "Unsigned 8 byte integer images are not supported " + "by the FITS standard" in str(e.value) + ) + else: + fits.write_image(data, header=header) + rdata = fits[-1].read() + + compare_array(data, rdata, "images") + + rh = fits[-1].read_header() + check_header(header, rh) + + if CFITSIO_VERSION >= 3.45: + with FITS(fname) as fits: + assert not fits[0].is_compressed(), 'not compressed' diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py new file mode 100644 index 0000000..ad72038 --- /dev/null +++ b/fitsio/tests/test_image_compression.py @@ -0,0 +1,670 @@ +import pytest +import sys +import os +import tempfile +from .checks import ( + # check_header, + compare_array, +) +import numpy as np +from ..fitslib import ( + FITS, + read, + write, + RICE_1, + SUBTRACTIVE_DITHER_1, + GZIP_1, + GZIP_2, + PLIO_1, + HCOMPRESS_1, +) +from ..util import cfitsio_is_bundled, cfitsio_version + +CFITSIO_VERSION = cfitsio_version(asfloat=True) + + +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize( + 'compress', + [ + 'rice', + 'hcompress', + 'plio', + 'gzip', + 'gzip_2', + 'gzip_lossless', + 'gzip_2_lossless', + ], +) +@pytest.mark.parametrize( + 'dtype', ['u1', 'i1', 'u2', 'i2', 'u4', 'i4', 'f4', 'f8'] +) +def test_compressed_write_read(compress, dtype, with_nan): + """ + Test writing and reading a rice compressed image + """ + nrows = 5 + ncols = 20 + if compress in ['rice', 'hcompress'] or 'gzip' in compress: + pass + elif compress == 'plio': + if dtype not in ['i1', 'i2', 'i4', 'f4', 'f8']: + return + else: + raise ValueError('unexpected compress %s' % compress) + + if 'lossless' in compress: + qlevel = None + else: + qlevel = 16 + + seed = 1919 + rng = np.random.RandomState(seed) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + if dtype[0] == 'f': + data = rng.normal(size=(nrows, ncols)) + if compress == 'plio': + data = data.clip(min=0) + data = data.astype(dtype) + else: + data = np.arange( + nrows * ncols, + dtype=dtype, + ).reshape(nrows, ncols) + + if "f" in dtype and with_nan and compress != "plio": + data[3, 11] = np.nan + + csend = compress.replace('_lossless', '') + write(fname, data, compress=csend, qlevel=qlevel) + rdata = read(fname, ext=1) + + if 'lossless' in compress or dtype[0] in ['i', 'u']: + np.testing.assert_array_equal( + data, + rdata, + err_msg="%s compressed images ('%s')" % (compress, dtype), + ) + else: + # lossy floating point + np.testing.assert_allclose( + data, + rdata, + rtol=0, + atol=0.2, + err_msg="%s compressed images ('%s')" % (compress, dtype), + ) + + with FITS(fname) as fits: + assert fits[1].is_compressed(), "is compressed" + + +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize( + 'compress', + [ + 'rice', + 'hcompress', + 'plio', + 'gzip', + 'gzip_2', + 'gzip_lossless', + 'gzip_2_lossless', + ], +) +@pytest.mark.parametrize( + 'dtype', ['u1', 'i1', 'u2', 'i2', 'u4', 'i4', 'f4', 'f8'] +) +def test_compressed_write_read_fitsobj(compress, dtype, with_nan): + """ + Test writing and reading a rice compressed image + + In this version, keep the fits object open + """ + + if ( + "gzip" in compress + and dtype in ["u2", "i2", "u4", "i4"] + and not cfitsio_is_bundled() + ): + pytest.xfail( + reason=( + "Non-bundled cfitsio libraries have a bug. " + "See https://github.com/HEASARC/cfitsio/pull/97." + ) + ) + + nrows = 5 + ncols = 20 + if compress in ['rice', 'hcompress'] or 'gzip' in compress: + pass + elif compress == 'plio': + if dtype not in ['i1', 'i2', 'i4', 'f4', 'f8']: + return + else: + raise ValueError('unexpected compress %s' % compress) + + if 'lossless' in compress: + qlevel = None + # qlevel = 9999 + else: + qlevel = 16 + + seed = 1919 + rng = np.random.RandomState(seed) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # note i8 not supported for compressed! + + if dtype[0] == 'f': + data = rng.normal(size=(nrows, ncols)) + if compress == 'plio': + data = data.clip(min=0) + data = data.astype(dtype) + else: + data = np.arange( + nrows * ncols, + dtype=dtype, + ).reshape(nrows, ncols) + + if "f" in dtype and with_nan and compress != "plio": + data[3, 11] = np.nan + + csend = compress.replace('_lossless', '') + fits.write_image(data, compress=csend, qlevel=qlevel) + rdata = fits[-1].read() + + if 'lossless' in compress or dtype[0] in ['i', 'u']: + # for integers we have chosen a wide range of values, so + # there will be no quantization and we expect no + # information loss + np.testing.assert_array_equal( + data, + rdata, + "%s compressed images ('%s')" % (compress, dtype), + ) + else: + # lossy floating point + np.testing.assert_allclose( + data, + rdata, + rtol=0, + atol=0.2, + err_msg="%s compressed images ('%s')" % (compress, dtype), + ) + + with FITS(fname) as fits: + assert fits[1].is_compressed(), "is compressed" + + +@pytest.mark.skipif(sys.version_info < (3, 9), reason='importlib bug in 3.8') +@pytest.mark.skipif(CFITSIO_VERSION < 3.49, reason='bug in cfitsio < 3.49') +def test_gzip_tile_compressed_read_lossless_astropy(): + """ + Test reading an image gzip compressed by astropy (fixed by cfitsio 3.49) + """ + import importlib.resources + + ref = ( + importlib.resources.files("fitsio") + / 'test_images' + / 'test_gzip_compressed_image.fits.fz' + ) # noqa + with importlib.resources.as_file(ref) as gzip_file: + data = read(gzip_file) + + compare_array(data, data * 0.0, "astropy lossless compressed image") + + +@pytest.mark.parametrize("with_nan", [False, True]) +def test_compress_preserve_zeros(with_nan): + """ + Test writing and reading gzip compressed image + """ + + zinds = [ + (1, 3), + (2, 9), + ] + + dtypes = ['f4', 'f8'] + + seed = 2020 + rng = np.random.RandomState(seed) + + # Do not test hcompress as it doesn't support SUBTRACTIVE_DITHER_2 + for compress in ['gzip', 'gzip_2', 'rice']: + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + for dtype in dtypes: + data = rng.normal(size=5 * 20).reshape(5, 20).astype(dtype) + for zind in zinds: + data[zind[0], zind[1]] = 0.0 + if with_nan: + data[3, 15] = np.nan + + fits.write_image( + data, + compress=compress, + qlevel=16, + qmethod='SUBTRACTIVE_DITHER_2', + ) + rdata = fits[-1].read() + + for zind in zinds: + assert rdata[zind[0], zind[1]] == 0.0 + if with_nan: + assert np.isnan(rdata[3, 15]) + + +@pytest.mark.parametrize("with_nan", [False, True]) +@pytest.mark.parametrize( + 'compress', + [ + 'rice', + 'hcompress', + 'plio', + ], +) +@pytest.mark.parametrize( + 'seed_type', + ['matched', 'unmatched', 'checksum', 'checksum_int'], +) +@pytest.mark.parametrize( + 'use_fits_object', + [False, True], +) +@pytest.mark.parametrize( + 'dtype', + ['f4', 'f8'], +) +def test_compressed_seed( + compress, seed_type, use_fits_object, dtype, with_nan +): + """ + Test writing and reading a rice compressed image + """ + nrows = 5 + ncols = 20 + + qlevel = 16 + + seed = 1919 + rng = np.random.RandomState(seed) + + if seed_type == 'matched': + # dither_seed = 9881 + dither_seed1 = 9881 + dither_seed2 = 9881 + elif seed_type == 'unmatched': + # dither_seed = None + dither_seed1 = 3 + dither_seed2 = 4 + elif seed_type == 'checksum': + dither_seed1 = 'checksum' + dither_seed2 = b'checksum' + elif seed_type == 'checksum_int': + dither_seed1 = -1 + # any negative means use checksum + dither_seed2 = -3 + + with tempfile.TemporaryDirectory() as tmpdir: + fname1 = os.path.join(tmpdir, 'test1.fits') + fname2 = os.path.join(tmpdir, 'test2.fits') + + data = rng.normal(size=(nrows, ncols)) + if compress == 'plio': + data = data.clip(min=0) + data = data.astype(dtype) + + if "f" in dtype and with_nan and compress != "plio": + data[3, 11] = np.nan + + if use_fits_object: + with FITS(fname1, 'rw') as fits1: + fits1.write( + data, + compress=compress, + qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed1, + ) + rdata1 = fits1[-1].read() + + with FITS(fname2, 'rw') as fits2: + fits2.write( + data, + compress=compress, + qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed2, + ) + rdata2 = fits2[-1].read() + else: + write( + fname1, + data, + compress=compress, + qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed1, + ) + rdata1 = read(fname1) + + write( + fname2, + data, + compress=compress, + qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed2, + ) + rdata2 = read(fname2) + + mess = "%s compressed images ('%s')" % (compress, dtype) + + if seed_type in ['checksum', 'checksum_int', 'matched']: + np.testing.assert_array_equal(rdata1, rdata2, mess) + else: + with pytest.raises(AssertionError): + np.testing.assert_array_equal(rdata1, rdata2, mess) + + if "f" in dtype and with_nan and compress != "plio": + assert np.isnan(rdata1[3, 11]) + assert np.isnan(rdata2[3, 11]) + else: + assert np.all(np.isfinite(rdata1)) + assert np.all(np.isfinite(rdata2)) + + +@pytest.mark.parametrize( + 'dither_seed', + ['blah', 10_001], +) +def test_compressed_seed_bad(dither_seed): + """ + Test writing and reading a rice compressed image + """ + compress = 'rice' + dtype = 'f4' + nrows = 5 + ncols = 20 + + qlevel = 16 + + seed = 1919 + rng = np.random.RandomState(seed) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + data = rng.normal(size=(nrows, ncols)) + data = data.astype(dtype) + + with pytest.raises(ValueError): + write( + fname, + data, + compress=compress, + qlevel=qlevel, + dither_seed=dither_seed, + ) + + +def test_memory_compressed_seed(): + import fitsio + + dtype = 'f4' + nrows = 300 + ncols = 500 + + seed = 1919 + rng = np.random.RandomState(seed) + + with tempfile.TemporaryDirectory() as tmpdir: + fname1 = os.path.join(tmpdir, 'test1.fits') + fname2 = os.path.join(tmpdir, 'test2.fits') + + data = rng.normal(size=(nrows, ncols)) + data = data.astype(dtype) + + fitsio.write( + fname1, + data.copy(), + dither_seed='checksum', + compress='RICE', + qlevel=1e-4, + tile_dims=(100, 100), + clobber=True, + ) + hdr = fitsio.read_header(fname1, ext=1) + dither1 = hdr['ZDITHER0'] + assert dither1 == 8269 + + fits = fitsio.FITS('mem://[compress R 100,100; qz -1e-4]', 'rw') + fits.write(data.copy(), dither_seed='checksum') + data = fits.read_raw() + fits.close() + f = open(fname2, 'wb') + f.write(data) + f.close() + hdr = fitsio.read_header(fname2, ext=1) + dither2 = hdr['ZDITHER0'] + assert dither1 == dither2 + + +def test_image_compression_inmem_subdither2(): + H, W = 100, 100 + rng = np.random.RandomState(seed=10) + img = rng.normal(size=(H, W)) + img[40:50, :] = 0.0 + with FITS('mem://[compress G 100,100; qz 0]', 'rw') as F: + F.write(img) + rawdata = F.read_raw() + + with tempfile.TemporaryDirectory() as tmpdir: + pth = os.path.join(tmpdir, 'out.fits') + with open(pth, 'wb') as f: + f.write(rawdata) + im2 = read(pth) + z = im2[40:50, :] + + minval = z.min() + assert minval == 0 + + +@pytest.mark.parametrize( + "kw,val", + [ + ("compress", RICE_1), + ("tile_dims", (10, 2)), + ("tile_dims", np.array([10, 2])), + ("tile_dims", [10, 2]), + ("qlevel", 10.0), + ("qmethod", SUBTRACTIVE_DITHER_1), + ("hcomp_scale", 10.0), + ("hcomp_smooth", True), + ], +) +@pytest.mark.parametrize("set_val_to_none", [False, True]) +def test_image_compression_raises_on_python_set(kw, val, set_val_to_none): + H, W = 100, 100 + rng = np.random.RandomState(seed=10) + img = rng.normal(size=(H, W)) + if set_val_to_none: + kws = {kw: None} + else: + kws = {kw: val} + + with FITS('mem://[compress G 100,100; qz 0]', 'rw') as F: + with pytest.raises(ValueError): + F.write(img, **kws) + + with FITS('mem://[compress G 100,100; qz 4.0]', 'rw') as F: + F.write(img, dither_seed=10) + + +@pytest.mark.parametrize( + "compress", + [ + RICE_1, + GZIP_1, + GZIP_2, + PLIO_1, + HCOMPRESS_1, + ], +) +@pytest.mark.parametrize( + "dtype", + [ + np.uint8, + np.int8, + np.uint16, + np.int16, + np.uint32, + np.int32, + ], +) +@pytest.mark.parametrize("fname", ["mem://", "test.fits"]) +def test_image_compression_inmem_lossess_int(compress, dtype, fname): + if not cfitsio_is_bundled(): + pytest.xfail( + reason=( + "Non-bundled cfitsio libraries have a bug. " + "See https://github.com/HEASARC/cfitsio/pull/97 " + "and https://github.com/HEASARC/cfitsio/pull/99." + ), + ) + if compress == PLIO_1 and dtype in [np.int16, np.uint32, np.int32]: + pytest.skip( + reason="PLIO lossless compression of int16, uint32, and " + "int32 types is not supported by cfitsio", + ) + rng = np.random.RandomState(seed=10) + img = rng.normal(size=(300, 300)) + if dtype in [ + np.uint8, + np.uint16, + np.uint32, + ]: + img = np.abs(img) + img = img.astype(dtype) + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, 'rw') as F: + F.write(img, compress=compress, qlevel=0) + rimg = F[-1].read() + assert rimg is not None + assert np.array_equal(rimg, img) + + +def test_image_compression_inmem_lossessgzip_int_zeros(): + img = np.zeros((300, 300)).astype(np.int32) + with FITS('mem://', 'rw') as F: + F.write(img, compress='GZIP', qlevel=0) + rimg = F[-1].read() + assert rimg is not None + assert np.array_equal(rimg, img) + + +def test_image_compression_inmem_lossessgzip_float(): + rng = np.random.RandomState(seed=10) + img = rng.normal(size=(300, 300)) + with FITS('mem://', 'rw') as F: + F.write(img, compress='GZIP', qlevel=0) + rimg = F[-1].read() + assert rimg is not None + assert np.array_equal(rimg, img) + + +def test_image_mem_reopen_noop(): + rng = np.random.RandomState(seed=10) + img = rng.normal(size=(300, 300)) + with FITS('mem://', 'rw') as F: + F.write(img) + rimg = F[-1].read() + assert rimg is not None + assert np.array_equal(rimg, img) + F.reopen() + rimg = F[-1].read() + assert rimg is not None + assert np.array_equal(rimg, img) + + with FITS('mem://', 'rw') as F: + F.write(img) + F.reopen() + rimg = F[-1].read() + assert rimg is not None + assert np.array_equal(rimg, img) + + +@pytest.mark.parametrize("nan_value", [np.nan, np.inf, -np.inf]) +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize( + "fname", + [ + "test.fits", + "mem://", + ], +) +def test_image_compression_nulls(fname, dtype, nan_value): + data = np.arange(36).reshape((6, 6)).astype(dtype) + data[1, 1] = nan_value + + # everything comes back as nan + if nan_value is not np.nan: + msk = ~np.isfinite(data) + cdata = data.copy() + cdata[msk] = np.nan + else: + cdata = data + + with tempfile.TemporaryDirectory() as tmpdir: + if "mem://" not in fname: + fpth = os.path.join(tmpdir, fname) + else: + fpth = fname + + with FITS(fpth, "rw") as fits: + fits.write( + data, + compress='RICE_1', + tile_dims=(3, 3), + dither_seed=10, + qlevel=2, + ) + read_data = fits[1].read() + + np.testing.assert_allclose( + read_data, + cdata, + ) + + if "mem://" not in fpth: + with FITS(fpth, "r") as fits: + read_data = fits[1].read() + np.testing.assert_allclose( + read_data, + cdata, + ) + + +if __name__ == '__main__': + test_compressed_seed( + compress='rice', + match_seed=False, + use_fits_object=True, + dtype='f4', + ) diff --git a/fitsio/tests/test_image_compression_defaults.py b/fitsio/tests/test_image_compression_defaults.py new file mode 100644 index 0000000..c7ee03e --- /dev/null +++ b/fitsio/tests/test_image_compression_defaults.py @@ -0,0 +1,287 @@ +import os +import tempfile +import numpy as np +import fitsio + + +def test_compression_nocompress(): + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write(img) + with fitsio.FITS(fn) as fits: + assert len(fits) == 1 + + +def test_compression_diskfile_kwargs(): + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write( + img, + compress='RICE', + tile_dims=(10, 5), + qlevel=7.0, + qmethod='SUBTRACTIVE_DITHER_2', + dither_seed=42, + ) + with fitsio.FITS(fn) as fits: + assert len(fits) == 2 + hdr = fitsio.read_header(fn, ext=1) + for key, val in [ + ('ZTILE1', 5), + ('ZTILE2', 10), + ('ZQUANTIZ', 'SUBTRACTIVE_DITHER_2'), + ('ZDITHER0', 42), + ('ZCMPTYPE', 'RICE_ONE'), + ]: + assert hdr[key] == val + + +def test_compression_efns(): + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS(fn + '[compress G]', 'rw', clobber=True) as fits: + fits.write(img) + hdr = fitsio.read_header(fn, ext=1) + for key, val in [ + ('ZTILE1', 20), + ('ZTILE2', 1), + ('ZQUANTIZ', 'SUBTRACTIVE_DITHER_1'), + ('ZCMPTYPE', 'GZIP_1'), + ]: + assert hdr[key] == val + + +def test_compression_efns_kwargs(): + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS( + fn + '[compress G 5 10; qz 8.0]', 'rw', clobber=True + ) as fits: + fits.write(img, dither_seed=42) + hdr = fitsio.read_header(fn, ext=1) + for key, val in [ + ('ZTILE1', 5), + ('ZTILE2', 10), + ('ZQUANTIZ', 'SUBTRACTIVE_DITHER_2'), + ('ZCMPTYPE', 'GZIP_1'), + ('ZDITHER0', 42), + ]: + assert hdr[key] == val + + +def test_compression_qlevels_none_zero(): + default_kws = { + "compress": fitsio.GZIP_2, + "tile_dims": np.array([100, 100]), + "qmethod": fitsio.SUBTRACTIVE_DITHER_2, + } + with tempfile.TemporaryDirectory() as tmpdir: + fnpat = os.path.join(tmpdir, 'test-%i.fits') + + H, W = 200, 200 + bigimg = np.random.uniform(size=(H, W)) + results = [] + # None: don't even use compression at all + # 0: lossless gzip + for i, qlevel in enumerate([None, 0, 16, 4, 1]): + fn = fnpat % i + ql = qlevel + kw = {} + kw.update(default_kws) + if ql is None: + kw.update(compress=0) + ql = 0 + kw["qlevel"] = ql + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write(bigimg, dither_seed=42, **kw) + filesize = os.stat(fn).st_size + img2 = fitsio.read(fn) + rms = np.sqrt(np.mean((img2 - bigimg) ** 2)) + results.append((qlevel, filesize, rms)) + # No compression + q, sz, rms = results[0] + assert sz == 2880 * (1 + int(np.ceil(H * W * 8 / 2880.0))) + assert rms == 0.0 + # GZIP lossless + q, sz, rms = results[1] + assert rms == 0.0 + # Decreasing file size + for r1, r2 in zip(results, results[1:]): + q1, sz1, rms1 = r1 + q2, sz2, rms2 = r2 + assert sz1 > sz2 + assert rms1 <= rms2 + + +def test_compression_hcomp_args(): + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS( + fn + '[compress HS 10 10; s 2.0]', 'rw', clobber=True + ) as fits: + fits.write(img, dither_seed=42) + hdr = fitsio.read_header(fn, ext=1) + for key, val in [ + ('ZTILE1', 10), + ('ZTILE2', 10), + ('ZQUANTIZ', 'SUBTRACTIVE_DITHER_1'), + ('ZCMPTYPE', 'HCOMPRESS_1'), + ('ZDITHER0', 42), + ('ZNAME1', 'SCALE'), + ('ZVAL1', 2.0), + ('ZNAME2', 'SMOOTH'), + ('ZVAL2', 1), + ]: + assert hdr[key] == val + + +def test_compression_qlevel_default(): + # Check that if not specified, qlevel defaults to 4. + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + H, W = 200, 200 + bigimg = np.random.uniform(size=(H, W)) + # Default qlevel + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write(bigimg, compress='GZIP') + size_def = os.stat(fn).st_size + hdr = fitsio.read_header(fn, ext=1) + print(hdr) + for key, val in [ + ('ZQUANTIZ', 'SUBTRACTIVE_DITHER_1'), + ('ZCMPTYPE', 'GZIP_1'), + ]: + assert hdr[key] == val + # qlevel=0 + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write(bigimg, compress='GZIP', qlevel=0) + size_0 = os.stat(fn).st_size + # qlevel=4 + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write(bigimg, compress='GZIP', qlevel=4) + size_4 = os.stat(fn).st_size + # qlevel=16 + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + fits.write(bigimg, compress='GZIP', qlevel=16) + size_16 = os.stat(fn).st_size + # zero means NO COMPRESSION + assert size_0 > size_4 + # heh, lower values mean MORE COMPRESSION + assert size_4 < size_16 + assert size_def == size_4 + + +def test_compression_multihdu_diskfile(): + # Check multi-HDU case with a normal file + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS(fn, 'rw', clobber=True) as fits: + # A + fits.write(img, extname='A') + # B + fits.write(img, extname='B', compress='GZIP') + # C + fits.write( + img, + extname='C', + compress='GZIP', + qmethod='SUBTRACTIVE_DITHER_2', + ) + # D + fits.write(img, extname='D') + # E + fits.write(img, extname='E', compress='GZIP') + # F + fits.write(img, extname='F', compress=None) + with fitsio.FITS(fn) as fits: + assert len(fits) == 6 + hdrA = fits['A'].read_header() + hdrB = fits['B'].read_header() + hdrC = fits['C'].read_header() + hdrD = fits['D'].read_header() + hdrE = fits['E'].read_header() + hdrF = fits['F'].read_header() + # A is uncompressed + assert 'ZCMPTYPE' not in hdrA + # B is gzip + assert hdrB['ZCMPTYPE'] == 'GZIP_1' + assert hdrB['ZQUANTIZ'] == 'SUBTRACTIVE_DITHER_1' + # C is gzip with SD2 + assert hdrC['ZCMPTYPE'] == 'GZIP_1' + assert hdrC['ZQUANTIZ'] == 'SUBTRACTIVE_DITHER_2' + # D is not compressed + assert 'ZCMPTYPE' not in hdrD + # E is GZIP again + assert hdrE['ZCMPTYPE'] == 'GZIP_1' + assert hdrE['ZQUANTIZ'] == 'SUBTRACTIVE_DITHER_1' + # F is not compressed + assert 'ZCMPTYPE' not in hdrF + + +def test_compression_multihdu_memfile(): + # Check multi-HDU case with a normal file + with tempfile.TemporaryDirectory() as tmpdir: + fn = os.path.join(tmpdir, 'test.fits') + + img = np.ones((20, 20)) + with fitsio.FITS("mem://", 'rw', clobber=True) as fits: + # A + fits.write(img, extname='A') + # B + fits.write(img, extname='B', compress='GZIP') + # C + fits.write( + img, + extname='C', + compress='GZIP', + qmethod='SUBTRACTIVE_DITHER_2', + ) + # D + fits.write(img, extname='D') + # E + fits.write(img, extname='E', compress='GZIP') + # F + fits.write(img, extname='F', compress=None) + + data = fits.read_raw() + with open(fn, 'wb') as f: + f.write(data) + + with fitsio.FITS(fn) as fits: + assert len(fits) == 6 + hdrA = fits['A'].read_header() + hdrB = fits['B'].read_header() + hdrC = fits['C'].read_header() + hdrD = fits['D'].read_header() + hdrE = fits['E'].read_header() + hdrF = fits['F'].read_header() + # A is uncompressed + assert 'ZCMPTYPE' not in hdrA + # B is gzip + assert hdrB['ZCMPTYPE'] == 'GZIP_1' + assert hdrB['ZQUANTIZ'] == 'SUBTRACTIVE_DITHER_1' + # C is gzip with SD2 + assert hdrC['ZCMPTYPE'] == 'GZIP_1' + assert hdrC['ZQUANTIZ'] == 'SUBTRACTIVE_DITHER_2' + # D is not compressed + assert 'ZCMPTYPE' not in hdrD + # E is GZIP again + assert hdrE['ZCMPTYPE'] == 'GZIP_1' + assert hdrE['ZQUANTIZ'] == 'SUBTRACTIVE_DITHER_1' + # F is not compressed + assert 'ZCMPTYPE' not in hdrF diff --git a/fitsio/tests/test_lib.py b/fitsio/tests/test_lib.py new file mode 100644 index 0000000..f00b34b --- /dev/null +++ b/fitsio/tests/test_lib.py @@ -0,0 +1,112 @@ +import os +import tempfile +import numpy as np +from ..fitslib import FITS, read_header +from .checks import compare_array, compare_rec + + +def test_move_by_name(): + """ + test moving hdus by name + """ + + nrows = 3 + + seed = 1234 + rng = np.random.RandomState(seed) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data1 = np.zeros(nrows, dtype=[('ra', 'f8'), ('dec', 'f8')]) + data1['ra'] = rng.uniform(nrows) + data1['dec'] = rng.uniform(nrows) + fits.write_table(data1, extname='mytable') + + fits[-1].write_key('EXTVER', 1) + + data2 = np.zeros(nrows, dtype=[('ra', 'f8'), ('dec', 'f8')]) + data2['ra'] = rng.uniform(nrows) + data2['dec'] = rng.uniform(nrows) + + fits.write_table(data2, extname='mytable') + fits[-1].write_key('EXTVER', 2) + + hdunum1 = fits.movnam_hdu('mytable', extver=1) + assert hdunum1 == 2 + hdunum2 = fits.movnam_hdu('mytable', extver=2) + assert hdunum2 == 3 + + +def test_ext_ver(): + """ + Test using extname and extver, all combinations I can think of + """ + + seed = 9889 + rng = np.random.RandomState(seed) + + dtype = [('num', 'i4'), ('ra', 'f8'), ('dec', 'f8')] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + img1 = np.arange(2 * 3, dtype='i4').reshape(2, 3) + 5 + img2 = np.arange(2 * 3, dtype='i4').reshape(2, 3) + 6 + img3 = np.arange(2 * 3, dtype='i4').reshape(2, 3) + 7 + + nrows = 3 + data1 = np.zeros(nrows, dtype=dtype) + + data1['num'] = 1 + data1['ra'] = rng.uniform(nrows) + data1['dec'] = rng.uniform(nrows) + + data2 = np.zeros(nrows, dtype=dtype) + + data2['num'] = 2 + data2['ra'] = rng.uniform(nrows) + data2['dec'] = rng.uniform(nrows) + + data3 = np.zeros(nrows, dtype=dtype) + data3['num'] = 3 + data3['ra'] = rng.uniform(nrows) + data3['dec'] = rng.uniform(nrows) + + hdr1 = {'k1': 'key1'} + hdr2 = {'k2': 'key2'} + + fits.write_image(img1, extname='myimage', header=hdr1, extver=1) + fits.write_table(data1) + fits.write_table(data2, extname='mytable', extver=1) + fits.write_image(img2, extname='myimage', header=hdr2, extver=2) + fits.write_table(data3, extname='mytable', extver=2) + fits.write_image(img3) + + d1 = fits[1].read() + d2 = fits['mytable'].read() + d2b = fits['mytable', 1].read() + d3 = fits['mytable', 2].read() + + for f in data1.dtype.names: + compare_rec(data1, d1, "data1") + compare_rec(data2, d2, "data2") + compare_rec(data2, d2b, "data2b") + compare_rec(data3, d3, "data3") + + dimg1 = fits[0].read() + dimg1b = fits['myimage', 1].read() + dimg2 = fits['myimage', 2].read() + dimg3 = fits[5].read() + + compare_array(img1, dimg1, "img1") + compare_array(img1, dimg1b, "img1b") + compare_array(img2, dimg2, "img2") + compare_array(img3, dimg3, "img3") + + rhdr1 = read_header(fname, ext='myimage', extver=1) + rhdr2 = read_header(fname, ext='myimage', extver=2) + assert 'k1' in rhdr1, 'testing k1 in header version 1' + assert 'k2' in rhdr2, 'testing k2 in header version 2' diff --git a/fitsio/tests/test_table.py b/fitsio/tests/test_table.py new file mode 100644 index 0000000..a5d900f --- /dev/null +++ b/fitsio/tests/test_table.py @@ -0,0 +1,1658 @@ +import pytest +import numpy as np +import os +import tempfile +from .checks import ( + compare_names, + compare_array, + compare_array_tol, + compare_object_array, + compare_rec, + compare_headerlist_header, + compare_rec_with_var, + compare_rec_subrows, +) +from .makedata import make_data +from ..fitslib import FITS, write, read +from .. import util +from .. import cfitsio_has_bzip2_support + +CFITSIO_VERSION = util.cfitsio_version(asfloat=True) +DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] +if CFITSIO_VERSION > 4: + DTYPES += ["u8"] + + +def test_table_read_write(): + adata = make_data() + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table( + adata['data'], header=adata['keys'], extname='mytable' + ) + + d = fits[1].read() + compare_rec(adata['data'], d, "table read/write") + + h = fits[1].read_header() + compare_headerlist_header(adata['keys'], h) + + # see if our convenience functions are working + write( + fname, + adata['data2'], + extname="newext", + header={'ra': 335.2, 'dec': -25.2}, + ) + d = read(fname, ext='newext') + compare_rec(adata['data2'], d, "table data2") + + # now test read_column + with FITS(fname) as fits: + for f in adata['data'].dtype.names: + d = fits[1].read_column(f) + compare_array( + adata['data'][f], d, "table 1 single field read '%s'" % f + ) + + for f in adata['data2'].dtype.names: + d = fits['newext'].read_column(f) + compare_array( + adata['data2'][f], d, "table 2 single field read '%s'" % f + ) + + # now list of columns + for cols in [ + ['u2scalar', 'f4vec', 'Sarr'], + ['f8scalar', 'u2arr', 'Sscalar'], + ]: + d = fits[1].read(columns=cols) + for f in d.dtype.names: + compare_array( + adata['data'][f][:], d[f], "test column list %s" % f + ) + + for rows in [[1, 3], [3, 1], [2, 2, 1]]: + d = fits[1].read(columns=cols, rows=rows) + for col in d.dtype.names: + compare_array( + adata['data'][col][rows], + d[col], + "test column list %s row subset" % col, + ) + for col in cols: + d = fits[1].read_column(col, rows=rows) + compare_array( + adata['data'][col][rows], + d, + "test column list %s row subset" % col, + ) + + +@pytest.mark.parametrize('nvec', [2, 1]) +def test_table_read_write_vec1(nvec): + """ + ensure the data for vec length 1 gets round-tripped, even though + the shape is not preserved + """ + dtype = [('x', 'f4', (nvec,))] + num = 10 + data = np.zeros(num, dtype=dtype) + data['x'] = np.arange(num * nvec).reshape(num, nvec) + assert data['x'].shape == (num, nvec) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data) + + d = fits[1].read() + if nvec == 1: + assert d['x'].shape == (num,) + compare_array( + data['x'].ravel(), + d['x'].ravel(), + "table single field read 'x'", + ) + + # see if our convenience functions are working + write( + fname, + data, + extname="newext", + ) + d = read(fname, ext='newext') + if nvec == 1: + assert d['x'].shape == (num,) + compare_array(data['x'].ravel(), d['x'].ravel(), "table data2") + + # now test read_column + with FITS(fname) as fits: + d = fits[1].read_column('x') + if nvec == 1: + assert d.shape == (num,) + compare_array( + data['x'].ravel(), d.ravel(), "table single field read 'x'" + ) + + +@pytest.mark.parametrize('nvec', [2, 1]) +def test_table_read_write_uvec1(nvec): + """ + ensure the data for U string vec length 1 gets round-tripped, even though + the shape is not preserved. Also test 2 for consistency + """ + + dtype = [('string', 'U10', (nvec,))] + num = 10 + data = np.zeros(num, dtype=dtype) + sravel = data['string'].ravel() + sravel[:] = ['%-10s' % i for i in range(num * nvec)] + assert data['string'].shape == (num, nvec) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data) + + d = fits[1].read() + + if nvec == 1: + assert d['string'].shape == (num,) + + compare_array( + data['string'].ravel(), + d['string'].ravel(), + "table single field read 'string'", + ) + + # see if our convenience functions are working + write( + fname, + data, + extname="newext", + ) + d = read(fname, ext='newext') + + if nvec == 1: + assert d['string'].shape == (num,) + compare_array( + data['string'].ravel(), + d['string'].ravel(), + "table data2", + ) + + # now test read_column + with FITS(fname) as fits: + d = fits[1].read_column('string') + + if nvec == 1: + assert d.shape == (num,) + compare_array( + data['string'].ravel(), + d.ravel(), + "table single field read 'string'", + ) + + +def test_table_column_index_scalar(): + """ + Test a basic table write, data and a header, then reading back in to + check the values + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.empty(1, dtype=[('Z', 'f8')]) + data['Z'][:] = 1.0 + fits.write_table(data) + fits.write_table(data) + + with FITS(fname, 'r') as fits: + assert fits[1]['Z'][0].ndim == 0 + assert fits[1][0].ndim == 0 + + +def test_table_read_empty_rows(): + """ + test reading empty list of rows from an table. + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.empty(1, dtype=[('Z', 'f8')]) + data['Z'][:] = 1.0 + fits.write_table(data) + fits.write_table(data) + + with FITS(fname, 'r') as fits: + assert len(fits[1].read(rows=[])) == 0 + assert len(fits[1].read(rows=range(0, 0))) == 0 + assert len(fits[1].read(rows=np.arange(0, 0))) == 0 + + +def test_table_format_column_subset(): + """ + Test a basic table write, data and a header, then reading back in to + check the values + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + data = np.empty(1, dtype=[('Z', 'f8'), ('Z_PERSON', 'f8')]) + data['Z'][:] = 1.0 + data['Z_PERSON'][:] = 1.0 + fits.write_table(data) + fits.write_table(data) + fits.write_table(data) + + with FITS(fname, 'r') as fits: + # assert we do not have an extra row of 'Z' + sz = str(fits[2]['Z_PERSON']).split('\n') + s = str(fits[2][('Z_PERSON', 'Z')]).split('\n') + assert len(sz) == len(s) - 1 + + +def test_table_write_dict_of_arrays_scratch(): + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + d = {} + for n in data.dtype.names: + d[n] = data[n] + + fits.write(d) + + d = read(fname) + compare_rec(data, d, "list of dicts, scratch") + + +def test_table_write_dict_of_arrays(): + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.create_table_hdu(data, extname='mytable') + + d = {} + for n in data.dtype.names: + d[n] = data[n] + + fits[-1].write(d) + + d = read(fname) + compare_rec(data, d, "list of dicts") + + +def test_table_write_dict_of_arrays_var(): + """ + This version creating the table from a dict of arrays, variable + lenght columns + """ + + adata = make_data() + vardata = adata['vardata'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + d = {} + for n in vardata.dtype.names: + d[n] = vardata[n] + + fits.write(d) + + d = read(fname) + compare_rec_with_var(vardata, d, "dict of arrays, var") + + +def test_table_write_list_of_arrays_scratch(): + """ + This version creating the table from the names and list, creating + table first + """ + + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + names = [n for n in data.dtype.names] + dlist = [data[n] for n in data.dtype.names] + fits.write(dlist, names=names) + + d = read(fname) + compare_rec(data, d, "list of arrays, scratch") + + +def test_table_write_list_of_arrays(): + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.create_table_hdu(data, extname='mytable') + + columns = [n for n in data.dtype.names] + dlist = [data[n] for n in data.dtype.names] + fits[-1].write(dlist, columns=columns) + + d = read(fname, ext='mytable') + compare_rec(data, d, "list of arrays") + + +def test_table_write_list_of_arrays_var(): + """ + This version creating the table from the names and list, variable + lenght cols + """ + adata = make_data() + vardata = adata['vardata'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + names = [n for n in vardata.dtype.names] + dlist = [vardata[n] for n in vardata.dtype.names] + fits.write(dlist, names=names) + + d = read(fname) + compare_rec_with_var(vardata, d, "list of arrays, var") + + +def test_table_write_bad_string(): + for d in ['S0', 'U0']: + dt = [('s', d)] + + # old numpy didn't allow this dtype, so will throw + # a TypeError for empty dtype + try: + data = np.zeros(1, dtype=dt) + supported = True + except TypeError: + supported = False + + if supported: + with pytest.raises(ValueError): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + with FITS(fname, 'rw') as fits: + fits.write(data) + + +def test_variable_length_columns(): + adata = make_data() + vardata = adata['vardata'] + + for vstorage in ['fixed', 'object']: + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw', vstorage=vstorage) as fits: + fits.write(vardata) + + # reading multiple columns + d = fits[1].read() + compare_rec_with_var( + vardata, d, "read all test '%s'" % vstorage + ) + + cols = ['u2scalar', 'Sobj'] + d = fits[1].read(columns=cols) + compare_rec_with_var( + vardata, d, "read all test subcols '%s'" % vstorage + ) + + # one at a time + for f in vardata.dtype.names: + d = fits[1].read_column(f) + if util.is_object(vardata[f]): + compare_object_array( + vardata[f], d, "read all field '%s'" % f + ) + + # same as above with slices + # reading multiple columns + d = fits[1][:] + compare_rec_with_var( + vardata, d, "read all test '%s'" % vstorage + ) + + d = fits[1][cols][:] + compare_rec_with_var( + vardata, d, "read all test subcols '%s'" % vstorage + ) + + # one at a time + for f in vardata.dtype.names: + d = fits[1][f][:] + if util.is_object(vardata[f]): + compare_object_array( + vardata[f], d, "read all field '%s'" % f + ) + + # + # now same with sub rows + # + + # reading multiple columns, sorted and unsorted + for rows in [[0, 2], [2, 0]]: + d = fits[1].read(rows=rows) + compare_rec_with_var( + vardata, + d, + "read subrows test '%s'" % vstorage, + rows=rows, + ) + + d = fits[1].read(columns=cols, rows=rows) + compare_rec_with_var( + vardata, + d, + "read subrows test subcols '%s'" % vstorage, + rows=rows, + ) + + # one at a time + for f in vardata.dtype.names: + d = fits[1].read_column(f, rows=rows) + if util.is_object(vardata[f]): + compare_object_array( + vardata[f], + d, + "read subrows field '%s'" % f, + rows=rows, + ) + + # same as above with slices + # reading multiple columns + d = fits[1][rows] + compare_rec_with_var( + vardata, + d, + "read subrows slice test '%s'" % vstorage, + rows=rows, + ) + d = fits[1][2:4] + compare_rec_with_var( + vardata, + d, + "read slice test '%s'" % vstorage, + rows=[2, 3], + ) + + d = fits[1][cols][rows] + compare_rec_with_var( + vardata, + d, + "read subcols subrows slice test '%s'" % vstorage, + rows=rows, + ) + + d = fits[1][cols][2:4] + + compare_rec_with_var( + vardata, + d, + "read subcols slice test '%s'" % vstorage, + rows=[2, 3], + ) + + # one at a time + for f in vardata.dtype.names: + d = fits[1][f][rows] + if util.is_object(vardata[f]): + compare_object_array( + vardata[f], + d, + "read subrows field '%s'" % f, + rows=rows, + ) + d = fits[1][f][2:4] + if util.is_object(vardata[f]): + compare_object_array( + vardata[f], + d, + "read slice field '%s'" % f, + rows=[2, 3], + ) + + +def test_table_iter(): + """ + Test iterating over rows of a table + """ + + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data, header=adata['keys'], extname='mytable') + + # one row at a time + with FITS(fname) as fits: + hdu = fits["mytable"] + i = 0 + for row_data in hdu: + compare_rec(data[i], row_data, "table data") + i += 1 + + +def test_ascii_table_write_read(): + """ + Test write and read for an ascii table + """ + + adata = make_data() + ascii_data = adata['ascii_data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table( + ascii_data, + table_type='ascii', + header=adata['keys'], + extname='mytable', + ) + + # cfitsio always reports type as i4 and f8, period, even if if + # written with higher precision. Need to fix that somehow + for f in ascii_data.dtype.names: + d = fits[1].read_column(f) + if d.dtype == np.float64: + # note we should be able to do 1.11e-16 in principle, but + # in practice we get more like 2.15e-16 + compare_array_tol( + ascii_data[f], d, 2.15e-16, "table field read '%s'" % f + ) + else: + compare_array( + ascii_data[f], d, "table field read '%s'" % f + ) + + for rows in [[1, 3], [3, 1]]: + for f in ascii_data.dtype.names: + d = fits[1].read_column(f, rows=rows) + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f][rows], + d, + 2.15e-16, + "table field read subrows '%s'" % f, + ) + else: + compare_array( + ascii_data[f][rows], + d, + "table field read subrows '%s'" % f, + ) + + beg = 1 + end = 3 + for f in ascii_data.dtype.names: + d = fits[1][f][beg:end] + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f][beg:end], + d, + 2.15e-16, + "table field read slice '%s'" % f, + ) + else: + compare_array( + ascii_data[f][beg:end], + d, + "table field read slice '%s'" % f, + ) + + cols = ['i2scalar', 'f4scalar'] + for f in ascii_data.dtype.names: + data = fits[1].read(columns=cols) + for f in data.dtype.names: + d = data[f] + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f], + d, + 2.15e-16, + "table subcol, '%s'" % f, + ) + else: + compare_array( + ascii_data[f], d, "table subcol, '%s'" % f + ) + + data = fits[1][cols][:] + for f in data.dtype.names: + d = data[f] + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f], + d, + 2.15e-16, + "table subcol, '%s'" % f, + ) + else: + compare_array( + ascii_data[f], d, "table subcol, '%s'" % f + ) + + for rows in [[1, 3], [3, 1]]: + for f in ascii_data.dtype.names: + data = fits[1].read(columns=cols, rows=rows) + for f in data.dtype.names: + d = data[f] + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f][rows], + d, + 2.15e-16, + "table subcol, '%s'" % f, + ) + else: + compare_array( + ascii_data[f][rows], + d, + "table subcol, '%s'" % f, + ) + + data = fits[1][cols][rows] + for f in data.dtype.names: + d = data[f] + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f][rows], + d, + 2.15e-16, + "table subcol/row, '%s'" % f, + ) + else: + compare_array( + ascii_data[f][rows], + d, + "table subcol/row, '%s'" % f, + ) + + for f in ascii_data.dtype.names: + data = fits[1][cols][beg:end] + for f in data.dtype.names: + d = data[f] + if d.dtype == np.float64: + compare_array_tol( + ascii_data[f][beg:end], + d, + 2.15e-16, + "table subcol/slice, '%s'" % f, + ) + else: + compare_array( + ascii_data[f][beg:end], + d, + "table subcol/slice, '%s'" % f, + ) + + +def test_table_insert_column(): + """ + Insert a new column + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data, header=adata['keys'], extname='mytable') + + d = fits[1].read() + + for n in d.dtype.names: + newname = n + '_insert' + + fits[1].insert_column(newname, d[n]) + + newdata = fits[1][newname][:] + + compare_array( + d[n], + newdata, + "table single field insert and read '%s'" % n, + ) + + +def test_table_delete_row_range(): + """ + Test deleting a range of rows using the delete_rows method + """ + + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data) + + rowslice = slice(1, 3) + with FITS(fname, 'rw') as fits: + fits[1].delete_rows(rowslice) + + with FITS(fname) as fits: + d = fits[1].read() + + compare_data = data[[0, 3]] + compare_rec(compare_data, d, "delete row range") + + +def test_table_delete_rows(): + """ + Test deleting specific set of rows using the delete_rows method + """ + + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data) + + rows2delete = [1, 3] + with FITS(fname, 'rw') as fits: + fits[1].delete_rows(rows2delete) + + with FITS(fname) as fits: + d = fits[1].read() + + compare_data = data[[0, 2]] + compare_rec(compare_data, d, "delete rows") + + +def test_table_where(): + """ + Use the where method to get indices for a row filter expression + """ + + adata = make_data() + data2 = adata['data2'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data2) + + # + # get all indices + # + with FITS(fname) as fits: + a = fits[1].where('x > 3 && y < 8') + b = np.where((data2['x'] > 3) & (data2['y'] < 8))[0] + np.testing.assert_array_equal(a, b) + + # + # get slice of indices + # + with FITS(fname) as fits: + a = fits[1].where('x > 3 && y < 8', 2, 8) + b = np.where((data2['x'][2:8] > 3) & (data2['y'][2:8] < 8))[0] + np.testing.assert_array_equal(a, b) + + +def test_table_resize(): + """ + Use the resize method to change the size of a table + + default values get filled in and these are tested + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + # + # shrink from back + # + with FITS(fname, 'rw', clobber=True) as fits: + fits.write_table(data) + + nrows = 2 + with FITS(fname, 'rw') as fits: + fits[1].resize(nrows) + + with FITS(fname) as fits: + d = fits[1].read() + + compare_data = data[0:nrows] + compare_rec(compare_data, d, "shrink from back") + + # + # shrink from front + # + with FITS(fname, 'rw', clobber=True) as fits: + fits.write_table(data) + + with FITS(fname, 'rw') as fits: + fits[1].resize(nrows, front=True) + + with FITS(fname) as fits: + d = fits[1].read() + + compare_data = data[nrows - data.size :] + compare_rec(compare_data, d, "shrink from front") + + # These don't get zerod + # the defaults below come out of cfitsio + # IDK where they are defined + nrows = 10 + add_data = np.zeros(nrows - data.size, dtype=data.dtype) + add_data['i1scalar'] = -(2**7) + add_data['i1vec'] = -(2**7) + add_data['i1arr'] = -(2**7) + add_data['u2scalar'] = 2**15 + add_data['u2vec'] = 2**15 + add_data['u2arr'] = 2**15 + add_data['u4scalar'] = 2**31 + add_data['u4vec'] = 2**31 + add_data['u4arr'] = 2**31 + if CFITSIO_VERSION > 4: + add_data['u8scalar'] = 2**63 + add_data['u8vec'] = 2**63 + add_data['u8arr'] = 2**63 + + # + # expand at the back + # + with FITS(fname, 'rw', clobber=True) as fits: + fits.write_table(data) + with FITS(fname, 'rw') as fits: + fits[1].resize(nrows) + + with FITS(fname) as fits: + d = fits[1].read() + + compare_data = np.hstack((data, add_data)) + compare_rec(compare_data, d, "expand at the back") + + # + # expand at the front + # + with FITS(fname, 'rw', clobber=True) as fits: + fits.write_table(data) + with FITS(fname, 'rw') as fits: + fits[1].resize(nrows, front=True) + + with FITS(fname) as fits: + d = fits[1].read() + + compare_data = np.hstack((add_data, data)) + # These don't get zerod + compare_rec(compare_data, d, "expand at the front") + + +def test_slice(): + """ + Test reading by slice + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # initial write + fits.write_table(data) + + # test reading single columns + for f in data.dtype.names: + d = fits[1][f][:] + compare_array( + data[f], d, "test read all rows %s column subset" % f + ) + + # test reading row subsets + rows = [1, 3] + for f in data.dtype.names: + d = fits[1][f][rows] + compare_array(data[f][rows], d, "test %s row subset" % f) + for f in data.dtype.names: + d = fits[1][f][1:3] + compare_array(data[f][1:3], d, "test %s row slice" % f) + for f in data.dtype.names: + d = fits[1][f][1:4:2] + compare_array( + data[f][1:4:2], d, "test %s row slice with step" % f + ) + for f in data.dtype.names: + d = fits[1][f][::2] + compare_array( + data[f][::2], d, "test %s row slice with only setp" % f + ) + + # now list of columns + cols = ['u2scalar', 'f4vec', 'Sarr'] + d = fits[1][cols][:] + for f in d.dtype.names: + compare_array(data[f][:], d[f], "test column list %s" % f) + + cols = ['u2scalar', 'f4vec', 'Sarr'] + d = fits[1][cols][rows] + for f in d.dtype.names: + compare_array( + data[f][rows], d[f], "test column list %s row subset" % f + ) + + cols = ['u2scalar', 'f4vec', 'Sarr'] + d = fits[1][cols][1:3] + for f in d.dtype.names: + compare_array( + data[f][1:3], d[f], "test column list %s row slice" % f + ) + + +def test_table_append(): + """ + Test creating a table and appending new rows. + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # initial write + fits.write_table(data, header=adata['keys'], extname='mytable') + # now append + data2 = data.copy() + data2['f4scalar'] = 3 + fits[1].append(data2) + + d = fits[1].read() + assert d.size == data.size * 2 + + compare_rec(data, d[0 : data.size], "Comparing initial write") + compare_rec(data2, d[data.size :], "Comparing appended data") + + h = fits[1].read_header() + compare_headerlist_header(adata['keys'], h) + + # append with list of arrays and names + names = data.dtype.names + data3 = [np.array(data[name]) for name in names] + fits[1].append(data3, names=names) + + d = fits[1].read() + assert d.size == data.size * 3 + compare_rec(data, d[2 * data.size :], "Comparing appended data") + + # append with list of arrays and columns + fits[1].append(data3, columns=names) + + d = fits[1].read() + assert d.size == data.size * 4 + compare_rec(data, d[3 * data.size :], "Comparing appended data") + + +def test_table_subsets(): + """ + testing reading subsets + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data, header=adata['keys'], extname='mytable') + + for rows in [[1, 3], [3, 1]]: + d = fits[1].read(rows=rows) + compare_rec_subrows(data, d, rows, "table subset") + columns = ['i1scalar', 'f4arr'] + d = fits[1].read(columns=columns, rows=rows) + + for f in columns: + d = fits[1].read_column(f, rows=rows) + compare_array( + data[f][rows], d, "row subset, multi-column '%s'" % f + ) + for f in data.dtype.names: + d = fits[1].read_column(f, rows=rows) + compare_array( + data[f][rows], d, "row subset, column '%s'" % f + ) + + +def test_gz_write_read(): + """ + Test a basic table write, data and a header, then reading back in to + check the values + + this code all works, but the file is zere size when done! + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data, header=adata['keys'], extname='mytable') + + d = fits[1].read() + compare_rec(data, d, "gzip write/read") + + h = fits[1].read_header() + for entry in adata['keys']: + name = entry['name'].upper() + value = entry['value'] + hvalue = h[name] + if isinstance(hvalue, str): + hvalue = hvalue.strip() + assert value == hvalue, "testing header key '%s'" % name + + if 'comment' in entry: + assert ( + entry['comment'].strip() == h.get_comment(name).strip() + ), "testing comment for header key '%s'" % name + + stat = os.stat(fname) + assert stat.st_size != 0, "Making sure the data was flushed to disk" + + +@pytest.mark.skipif( + not cfitsio_has_bzip2_support(), + reason='cfitsio was not built with bzip2 support', +) +def test_bz2_read(): + ''' + Write a normal .fits file, run bzip2 on it, then read the bz2 + file and verify that it's the same as what we put in; we don't + [currently support or] test *writing* bzip2. + ''' + + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + bzfname = fname + '.bz2' + + try: + fits = FITS(fname, 'rw') + fits.write_table(data, header=adata['keys'], extname='mytable') + fits.close() + + os.system('bzip2 %s' % fname) + f2 = FITS(bzfname) + d = f2[1].read() + compare_rec(data, d, "bzip2 read") + + h = f2[1].read_header() + for entry in adata['keys']: + name = entry['name'].upper() + value = entry['value'] + hvalue = h[name] + if isinstance(hvalue, str): + hvalue = hvalue.strip() + + assert value == hvalue, "testing header key '%s'" % name + + if 'comment' in entry: + assert ( + entry['comment'].strip() == h.get_comment(name).strip() + ), "testing comment for header key '%s'" % name + except Exception: + import traceback + + traceback.print_exc() + + assert False, 'Exception in testing bzip2 reading' + + +def test_checksum(): + """ + test that checksumming works + """ + adata = make_data() + data = adata['data'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(data, header=adata['keys'], extname='mytable') + fits[1].write_checksum() + fits[1].verify_checksum() + + +def test_trim_strings(): + """ + test mode where we strim strings on read + """ + + dt = [('fval', 'f8'), ('name', 'S15'), ('vec', 'f4', 2)] + n = 3 + data = np.zeros(n, dtype=dt) + data['fval'] = np.random.random(n) + data['vec'] = np.random.random(n * 2).reshape(n, 2) + + data['name'] = ['mike', 'really_long_name_to_fill', 'jan'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write(data) + + for onconstruct in [True, False]: + if onconstruct: + ctrim = True + otrim = False + else: + ctrim = False + otrim = True + + with FITS(fname, 'rw', trim_strings=ctrim) as fits: + if ctrim: + dread = fits[1][:] + compare_rec( + data, + dread, + "trimmed strings constructor", + ) + + dname = fits[1]['name'][:] + compare_array( + data['name'], + dname, + "trimmed strings col read, constructor", + ) + dread = fits[1][['name']][:] + compare_array( + data['name'], + dread['name'], + "trimmed strings col read, constructor", + ) + + dread = fits[1].read(trim_strings=otrim) + compare_rec( + data, + dread, + "trimmed strings keyword", + ) + dname = fits[1].read(columns='name', trim_strings=otrim) + compare_array( + data['name'], + dname, + "trimmed strings col keyword", + ) + dread = fits[1].read(columns=['name'], trim_strings=otrim) + compare_array( + data['name'], + dread['name'], + "trimmed strings col keyword", + ) + + # convenience function + dread = read(fname, trim_strings=True) + compare_rec( + data, + dread, + "trimmed strings convenience function", + ) + dname = read(fname, columns='name', trim_strings=True) + compare_array( + data['name'], + dname, + "trimmed strings col convenience function", + ) + dread = read(fname, columns=['name'], trim_strings=True) + compare_array( + data['name'], + dread['name'], + "trimmed strings col convenience function", + ) + + +def test_lower_upper(): + """ + test forcing names to upper and lower + """ + + rng = np.random.RandomState(8908) + + dt = [('MyName', 'f8'), ('StuffThings', 'i4'), ('Blah', 'f4')] + data = np.zeros(3, dtype=dt) + data['MyName'] = rng.uniform(data.size) + data['StuffThings'] = rng.uniform(data.size) + data['Blah'] = rng.uniform(data.size) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write(data) + + for i in [1, 2]: + if i == 1: + lower = True + upper = False + else: + lower = False + upper = True + + with FITS(fname, 'rw', lower=lower, upper=upper) as fits: + for rows in [None, [1, 2]]: + d = fits[1].read(rows=rows) + compare_names( + d.dtype.names, + data.dtype.names, + lower=lower, + upper=upper, + ) + + d = fits[1].read( + rows=rows, columns=['MyName', 'stuffthings'] + ) + compare_names( + d.dtype.names, + data.dtype.names[0:2], + lower=lower, + upper=upper, + ) + + d = fits[1][1:2] + compare_names( + d.dtype.names, + data.dtype.names, + lower=lower, + upper=upper, + ) + + if rows is not None: + d = fits[1][rows] + else: + d = fits[1][:] + + compare_names( + d.dtype.names, + data.dtype.names, + lower=lower, + upper=upper, + ) + + if rows is not None: + d = fits[1][['myname', 'stuffthings']][rows] + else: + d = fits[1][['myname', 'stuffthings']][:] + + compare_names( + d.dtype.names, + data.dtype.names[0:2], + lower=lower, + upper=upper, + ) + + # using overrides + with FITS(fname, 'rw') as fits: + for rows in [None, [1, 2]]: + d = fits[1].read(rows=rows, lower=lower, upper=upper) + compare_names( + d.dtype.names, + data.dtype.names, + lower=lower, + upper=upper, + ) + + d = fits[1].read( + rows=rows, + columns=['MyName', 'stuffthings'], + lower=lower, + upper=upper, + ) + compare_names( + d.dtype.names, + data.dtype.names[0:2], + lower=lower, + upper=upper, + ) + + for rows in [None, [1, 2]]: + d = read(fname, rows=rows, lower=lower, upper=upper) + compare_names( + d.dtype.names, data.dtype.names, lower=lower, upper=upper + ) + + d = read( + fname, + rows=rows, + columns=['MyName', 'stuffthings'], + lower=lower, + upper=upper, + ) + compare_names( + d.dtype.names, + data.dtype.names[0:2], + lower=lower, + upper=upper, + ) + + +def test_read_raw(): + """ + testing reading the file as raw bytes + """ + rng = np.random.RandomState(8908) + + dt = [('MyName', 'f8'), ('StuffThings', 'i4'), ('Blah', 'f4')] + data = np.zeros(3, dtype=dt) + data['MyName'] = rng.uniform(data.size) + data['StuffThings'] = rng.uniform(data.size) + data['Blah'] = rng.uniform(data.size) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + try: + with FITS(fname, 'rw') as fits: + fits.write(data) + raw1 = fits.read_raw() + + with FITS('mem://', 'rw') as fits: + fits.write(data) + raw2 = fits.read_raw() + + with open(fname, 'rb') as fobj: + raw3 = fobj.read() + + assert raw1 == raw2 + assert raw1 == raw3 + except Exception: + import traceback + + traceback.print_exc() + assert False, 'Exception in testing read_raw' + + +def test_table_bitcol_read_write(): + """ + Test basic write/read with bitcols + """ + + adata = make_data() + bdata = adata['bdata'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.write_table(bdata, extname='mytable', write_bitcols=True) + + d = fits[1].read() + compare_rec(bdata, d, "table read/write") + + rows = [0, 2] + d = fits[1].read(rows=rows) + compare_rec(bdata[rows], d, "table read/write rows") + + d = fits[1][:2] + compare_rec(bdata[:2], d, "table read/write slice") + + # now test read_column + with FITS(fname) as fits: + for f in bdata.dtype.names: + d = fits[1].read_column(f) + compare_array( + bdata[f], d, "table 1 single field read '%s'" % f + ) + + # now list of columns + for cols in [['b1vec', 'b1arr']]: + d = fits[1].read(columns=cols) + for f in d.dtype.names: + compare_array(bdata[f][:], d[f], "test column list %s" % f) + + for rows in [[1, 3], [3, 1]]: + d = fits[1].read(columns=cols, rows=rows) + for f in d.dtype.names: + compare_array( + bdata[f][rows], + d[f], + "test column list %s row subset" % f, + ) + + +def test_table_bitcol_append(): + """ + Test creating a table with bitcol support and appending new rows. + """ + adata = make_data() + bdata = adata['bdata'] + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # initial write + fits.write_table(bdata, extname='mytable', write_bitcols=True) + + with FITS(fname, 'rw') as fits: + # now append + bdata2 = bdata.copy() + fits[1].append(bdata2) + + d = fits[1].read() + assert d.size == bdata.size * 2 + + compare_rec(bdata, d[0 : bdata.size], "Comparing initial write") + compare_rec(bdata2, d[bdata.size :], "Comparing appended data") + + +def test_table_bitcol_insert(): + """ + Test creating a table with bitcol support and appending new rows. + """ + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + # initial write + nrows = 3 + d = np.zeros(nrows, dtype=[('ra', 'f8')]) + d['ra'] = range(d.size) + fits.write(d) + + with FITS(fname, 'rw') as fits: + bcol = np.array([True, False, True]) + + # now append + fits[-1].insert_column( + 'bscalar_inserted', bcol, write_bitcols=True + ) + + d = fits[-1].read() + assert d.size == nrows, 'read size equals' + compare_array(bcol, d['bscalar_inserted'], "inserted bitcol") + + bvec = np.array([[True, False], [False, True], [True, True]]) + + # now append + fits[-1].insert_column('bvec_inserted', bvec, write_bitcols=True) + + d = fits[-1].read() + assert d.size == nrows, 'read size equals' + compare_array(bvec, d['bvec_inserted'], "inserted bitcol") + + +def test_table_write_dict_of_arrays_unaligned(): + data = {} + for dtype in DTYPES: + _data = np.arange(20, dtype=dtype) + # The code to make the unaligned view was generated + # by Google's AI and then modified by hand to fix a bug. + unaligned_data = np.ndarray( + shape=(19,), + dtype=_data.dtype, + buffer=_data.data, + offset=1, # Offset by 1 byte + strides=_data.strides, + ) + if not dtype.endswith("1"): + assert not unaligned_data.flags["ALIGNED"] + + data[dtype.replace("<", "l")] = unaligned_data + + dtype = np.dtype( + { + "names": list(data.keys()), + "formats": [v.dtype for v in data.values()], + } + ) + data_stra = np.zeros(data[dtype.names[0]].shape, dtype=dtype) + for k, v in data.items(): + data_stra[k] = v + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + fits.create_table_hdu(data, extname='mytable') + fits[-1].write(data) + + d = read(fname) + compare_rec(data_stra, d, "list of dicts") + + +@pytest.mark.parametrize("table_type", ["binary", "ascii"]) +def test_table_big_col(table_type): + d = np.ones(1, dtype=[("blah", "U70000")]) + d["blah"] = "".join(["a"] * 60000) + with tempfile.TemporaryDirectory() as tmpdir: + pth = os.path.join(tmpdir, "test.fits") + # v3 cfitsio that is not bundled fails for big + # columns + if table_type == "ascii" or CFITSIO_VERSION < 4: + with pytest.raises(OSError) as e: + write(pth, d, table_type=table_type) + assert "FITSIO status = 236: column exceeds width of table" in str( + e.value + ) + assert ( + "string column is too wide: 70000; " + "max supported width is" in str(e.value) + ) + else: + write(pth, d, table_type=table_type) + data = read(pth) + np.testing.assert_array_equal(d, data) + + +@pytest.mark.xfail( + condition=CFITSIO_VERSION < 4, + reason=( + "cfitsio versions < 4 do not easily support null-terminated strings" + ), +) +@pytest.mark.parametrize("table_type", ["binary", "ascii"]) +def test_table_null_end_strings(table_type): + d = np.ones(2, dtype=[("blah", "U70")]) + d["blah"][0] = "".join(["a"] * 60) + d["blah"][1] = "" + with tempfile.TemporaryDirectory() as tmpdir: + pth = os.path.join(tmpdir, "test.fits") + write(pth, d, table_type=table_type) + data = read(pth) + assert len(data["blah"][0]) == 60 + assert "U70" in data["blah"].dtype.descr[0][1] + + if table_type == "ascii": + # null strings in ascii tables are a single blank + d["blah"][1] = " " + np.testing.assert_array_equal(d, data) + + +def test_table_read_write_ulonglong(): + adata = np.zeros(5, dtype=[("u8scalar", "u8")]) + adata["u8scalar"] = (2**64 - 1) - np.arange(5, dtype="u8") + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + if CFITSIO_VERSION < 3.45: + with pytest.raises(IOError) as e: + fits.write_table( + adata, + extname='mytable', + ) + assert "'W'" in str(e.value) + else: + fits.write_table( + adata, + extname='mytable', + ) + d = fits[1].read() + compare_rec(adata, d, "table read/write") + + +@pytest.mark.parametrize("typ", ["u8", "u4", "i8"]) +def test_table_read_write_ulonglong_ascii_raises(typ): + adata = np.zeros(5, dtype=[("scalar", typ)]) + if typ == "u8": + val = 2**64 - 1 + elif typ == "u4": + val = 2**32 - 1 + elif typ == "i8": + val = 2**31 - 1 + adata["scalar"] = (val) - np.arange(5, dtype=typ) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + with FITS(fname, 'rw') as fits: + with pytest.raises(ValueError) as e: + fits.write_table( + adata, + extname='mytable', + table_type='ascii', + ) + assert f"unsupported type '{typ}' for ascii tables" in str(e.value) diff --git a/fitsio/tests/test_util.py b/fitsio/tests/test_util.py new file mode 100644 index 0000000..7fd76b9 --- /dev/null +++ b/fitsio/tests/test_util.py @@ -0,0 +1,78 @@ +import math +import numpy as np + +import pytest + +from ..util import ( + _nonfinite_as_cfitsio_floating_null_value, + cfitsio_version, + _FLOATING_NULL_VALUE, +) + +CFITSIO_VERSION = cfitsio_version(asfloat=True) +DTYPES = ['u1', 'i1', 'u2', 'i2', 'f4', 'f8'] +if CFITSIO_VERSION > 3.44: + DTYPES += ["u8"] + + +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("with_nan", [True, False]) +@pytest.mark.parametrize("hdu_is_compressed", [True, False]) +def test_nonfinite_as_cfitsio_floating_null_value( + dtype, with_nan, hdu_is_compressed +): + data = np.arange(5 * 20, dtype=dtype).reshape(5, 20) + if "f" in dtype and with_nan: + data[3, 13] = np.nan + data[1, 7] = np.inf + data[1, 9] = -np.inf + + with _nonfinite_as_cfitsio_floating_null_value( + data, hdu_is_compressed + ) as ( + nan_data, + any_nan, + ): + if with_nan and "f" in dtype and hdu_is_compressed: + assert any_nan + assert not np.any(np.isnan(nan_data)) + msk = ~np.isfinite(data) + np.testing.assert_array_equal(data[msk], _FLOATING_NULL_VALUE) + np.testing.assert_array_equal(data[~msk], nan_data[~msk]) + else: + assert not any_nan + np.testing.assert_array_equal(nan_data, data) + + if with_nan and "f" in dtype: + np.testing.assert_array_equal(data[3, 13], np.nan) + np.testing.assert_array_equal(data[1, 7], np.inf) + np.testing.assert_array_equal(data[1, 9], -np.inf) + + +def test_cfitsio_floating_null_value_equal_inf(): + assert np.float64(np.inf) == _FLOATING_NULL_VALUE + assert np.float32(np.inf) == _FLOATING_NULL_VALUE + assert np.inf == _FLOATING_NULL_VALUE + assert math.inf == _FLOATING_NULL_VALUE + + +def test_nonfinite_as_cfitsio_floating_null_value_with_exception(): + data = np.arange(5 * 20, dtype=np.float32).reshape(5, 20) + data[3, 13] = np.nan + data[1, 7] = np.inf + data[1, 9] = -np.inf + + with pytest.raises(RuntimeError): + with _nonfinite_as_cfitsio_floating_null_value(data, True) as ( + nan_data, + any_nan, + ): + assert any_nan + assert not np.any(np.isnan(nan_data)) + msk = ~np.isfinite(data) + np.testing.assert_array_equal(data[msk], _FLOATING_NULL_VALUE) + np.testing.assert_array_equal(data[~msk], nan_data[~msk]) + raise RuntimeError("Exception raised while data is modified!") + + assert data[1, 7] == np.inf + assert data[1, 9] == -np.inf diff --git a/fitsio/tests/test_warnings.py b/fitsio/tests/test_warnings.py new file mode 100644 index 0000000..3a6d50b --- /dev/null +++ b/fitsio/tests/test_warnings.py @@ -0,0 +1,42 @@ +import os +import tempfile +import warnings +import numpy as np +from ..fitslib import FITS +from ..util import FITSRuntimeWarning + + +def test_non_standard_key_value(): + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + im = np.zeros((3, 3)) + with warnings.catch_warnings(record=True) as w: + with FITS(fname, 'rw') as fits: + fits.write(im) + + # now write a key with a non-standard value + value = {'test': 3} + fits[-1].write_key('odd', value) + + # DeprecationWarnings have crept into the Warning list. This will + # filter the list to be just + # FITSRuntimeWarning instances. + # @at88mph 2019.10.09 + filtered_warnings = list( + filter( + lambda x: 'FITSRuntimeWarning' in '{}'.format(x.category), + w, + ) # noqa + ) + + assert len(filtered_warnings) == 1, ( + 'Wrong length of output (Expected {} but got {}.)'.format( + 1, + len(filtered_warnings), + ) + ) + assert issubclass( + filtered_warnings[-1].category, + FITSRuntimeWarning, + ) diff --git a/fitsio/util.py b/fitsio/util.py new file mode 100644 index 0000000..197c6bd --- /dev/null +++ b/fitsio/util.py @@ -0,0 +1,221 @@ +""" +utilities for the fits library +""" + +from contextlib import contextmanager +import sys +import numpy + +from . import _fitsio_wrap + +if sys.version_info >= (3, 0, 0): + IS_PY3 = True +else: + IS_PY3 = False + +_FLOATING_NULL_VALUE = _fitsio_wrap.cfitsio_null_value_for_nan() + + +class FITSRuntimeWarning(RuntimeWarning): + pass + + +def cfitsio_version(asfloat=False): + """ + Return the cfitsio version as a string. + """ + # use string version to avoid roundoffs + ver = '%0.3f' % _fitsio_wrap.cfitsio_version() + if asfloat: + return float(ver) + else: + return ver + + +def cfitsio_is_bundled(): + """Return True if library was built with a + bundled copy of cfitsio. + """ + return _fitsio_wrap.cfitsio_is_bundled() + + +if sys.version_info > (3, 0, 0): + _itypes = (int,) + _stypes = (str, bytes) +else: + _itypes = (int, long) # noqa - only for py2 + _stypes = ( + basestring, # noqa - only for py2 + unicode, # noqa - only for py2 + ) # noqa - only for py2 + +_itypes += ( + numpy.uint8, + numpy.int8, + numpy.uint16, + numpy.int16, + numpy.uint32, + numpy.int32, + numpy.uint64, + numpy.int64, +) + +# different for py3 +if numpy.lib.NumpyVersion(numpy.__version__) < "1.28.0": + _stypes += ( + numpy.string_, + numpy.str_, + ) +else: + _stypes += ( + numpy.bytes_, + numpy.str_, + ) + +# for header keywords +_ftypes = (float, numpy.float32, numpy.float64) + + +def isstring(arg): + return isinstance(arg, _stypes) + + +def isinteger(arg): + return isinstance(arg, _itypes) + + +def is_object(arr): + if arr.dtype.descr[0][1][1] == 'O': + return True + else: + return False + + +def fields_are_object(arr): + isobj = numpy.zeros(len(arr.dtype.names), dtype=bool) + for i, name in enumerate(arr.dtype.names): + if is_object(arr[name]): + isobj[i] = True + return isobj + + +def is_little_endian(array): + """ + Return True if array is little endian, False otherwise. + + Parameters + ---------- + array: numpy array + A numerical python array. + + Returns + ------- + Truth value: + True for little-endian + + Notes + ----- + Strings are neither big or little endian. The input must be a simple numpy + array, not an array with fields. + """ + if numpy.little_endian: + machine_little = True + else: + machine_little = False + + byteorder = array.dtype.base.byteorder + return (byteorder == '<') or (machine_little and byteorder == '=') + + +def array_to_native(array, inplace=False): + """ + Convert an array to the native byte order. + + NOTE: the inplace keyword argument is not currently used. + """ + if numpy.little_endian: + machine_little = True + else: + machine_little = False + + data_little = False + if array.dtype.names is None: + if array.dtype.base.byteorder == '|': + # strings and 1 byte integers + return array + + data_little = is_little_endian(array) + else: + # assume all are same byte order: we only need to find one with + # little endian + for fname in array.dtype.names: + if is_little_endian(array[fname]): + data_little = True + break + + if (machine_little and not data_little) or ( + not machine_little and data_little + ): + output = array.byteswap(inplace) + else: + output = array + + return numpy.require(output, requirements=['ALIGNED']) + + +if numpy.lib.NumpyVersion(numpy.__version__) >= "2.0.0": + copy_if_needed = None +elif numpy.lib.NumpyVersion(numpy.__version__) < "1.28.0": + copy_if_needed = False +else: + # 2.0.0 dev versions, handle cases where copy may or may not exist + try: + numpy.array([1]).__array__(copy=None) + copy_if_needed = None + except TypeError: + copy_if_needed = False + + +def array_to_native_c(array_in, inplace=False): + # copy only made if not C order + arr = numpy.require( + array_in, + requirements=['C_CONTIGUOUS', 'ALIGNED'], + ) + return array_to_native(arr, inplace=inplace) + + +def mks(val): + """ + make sure the value is a string, paying mind to python3 vs 2 + """ + if sys.version_info > (3, 0, 0): + if isinstance(val, bytes): + sval = str(val, 'utf-8') + else: + sval = str(val) + else: + sval = str(val) + + return sval + + +@contextmanager +def _nonfinite_as_cfitsio_floating_null_value(data, target_hdu_compressed): + try: + has_nonfinite = False + if ( + data is not None + and data.dtype.kind == "f" + and target_hdu_compressed + ): + msk_nonfinite = ~numpy.isfinite(data) + if numpy.any(msk_nonfinite): + has_nonfinite = True + old_vals = data[msk_nonfinite] + data[msk_nonfinite] = _FLOATING_NULL_VALUE + + yield data, has_nonfinite + finally: + if has_nonfinite: + data[msk_nonfinite] = old_vals diff --git a/patches/Makefile.am.patch b/patches/Makefile.am.patch new file mode 100644 index 0000000..be193b2 --- /dev/null +++ b/patches/Makefile.am.patch @@ -0,0 +1,14 @@ +--- a/cfitsio-4.6.3/Makefile.am ++++ b/cfitsio-4.6.3/Makefile.am +@@ -82,7 +82,10 @@ libcfitsio_la_SOURCES = \ + fits_hdecompress.c \ + simplerng.c \ + zcompress.c \ +- zuncompress.c ++ zuncompress.c \ ++ adler32.c crc32.c deflate.c infback.c \ ++ inffast.c inflate.c inftrees.c trees.c \ ++ uncompr.c zutil.c + + if !NOFORTRAN + libcfitsio_la_SOURCES += $(F77_WRAPPERS) diff --git a/patches/Makefile.in.patch b/patches/Makefile.in.patch new file mode 100644 index 0000000..09acae6 --- /dev/null +++ b/patches/Makefile.in.patch @@ -0,0 +1,267 @@ +--- a/cfitsio-4.6.3/Makefile.in ++++ b/cfitsio-4.6.3/Makefile.in +@@ -166,8 +166,9 @@ am__libcfitsio_la_SOURCES_DIST = buffers.c cfileio.c checksum.c \ + putcoluj.c putkey.c region.c scalnull.c swapproc.c wcssub.c \ + wcsutil.c imcompress.c quantize.c ricecomp.c pliocomp.c \ + fits_hcompress.c fits_hdecompress.c simplerng.c zcompress.c \ +- zuncompress.c f77_wrap1.c f77_wrap2.c f77_wrap3.c f77_wrap4.c \ +- drvrgsiftp.c ++ zuncompress.c adler32.c crc32.c deflate.c infback.c inffast.c \ ++ inflate.c inftrees.c trees.c uncompr.c zutil.c f77_wrap1.c \ ++ f77_wrap2.c f77_wrap3.c f77_wrap4.c drvrgsiftp.c + am__objects_1 = libcfitsio_la-f77_wrap1.lo libcfitsio_la-f77_wrap2.lo \ + libcfitsio_la-f77_wrap3.lo libcfitsio_la-f77_wrap4.lo + @NOFORTRAN_FALSE@am__objects_2 = $(am__objects_1) +@@ -204,6 +205,11 @@ am_libcfitsio_la_OBJECTS = libcfitsio_la-buffers.lo \ + libcfitsio_la-pliocomp.lo libcfitsio_la-fits_hcompress.lo \ + libcfitsio_la-fits_hdecompress.lo libcfitsio_la-simplerng.lo \ + libcfitsio_la-zcompress.lo libcfitsio_la-zuncompress.lo \ ++ libcfitsio_la-adler32.lo libcfitsio_la-crc32.lo \ ++ libcfitsio_la-deflate.lo libcfitsio_la-infback.lo \ ++ libcfitsio_la-inffast.lo libcfitsio_la-inflate.lo \ ++ libcfitsio_la-inftrees.lo libcfitsio_la-trees.lo \ ++ libcfitsio_la-uncompr.lo libcfitsio_la-zutil.lo \ + $(am__objects_2) $(am__objects_4) + libcfitsio_la_OBJECTS = $(am_libcfitsio_la_OBJECTS) + AM_V_lt = $(am__v_lt_@AM_V@) +@@ -279,9 +285,12 @@ am__v_at_1 = + DEFAULT_INCLUDES = -I.@am__isrc@ + depcomp = $(SHELL) $(top_srcdir)/config/depcomp + am__maybe_remake_depfiles = depfiles +-am__depfiles_remade = ./$(DEPDIR)/libcfitsio_la-buffers.Plo \ ++am__depfiles_remade = ./$(DEPDIR)/libcfitsio_la-adler32.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-buffers.Plo \ + ./$(DEPDIR)/libcfitsio_la-cfileio.Plo \ + ./$(DEPDIR)/libcfitsio_la-checksum.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-crc32.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-deflate.Plo \ + ./$(DEPDIR)/libcfitsio_la-drvrfile.Plo \ + ./$(DEPDIR)/libcfitsio_la-drvrgsiftp.Plo \ + ./$(DEPDIR)/libcfitsio_la-drvrmem.Plo \ +@@ -317,6 +326,10 @@ am__depfiles_remade = ./$(DEPDIR)/libcfitsio_la-buffers.Plo \ + ./$(DEPDIR)/libcfitsio_la-grparser.Plo \ + ./$(DEPDIR)/libcfitsio_la-histo.Plo \ + ./$(DEPDIR)/libcfitsio_la-imcompress.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-infback.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-inffast.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-inflate.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-inftrees.Plo \ + ./$(DEPDIR)/libcfitsio_la-iraffits.Plo \ + ./$(DEPDIR)/libcfitsio_la-modkey.Plo \ + ./$(DEPDIR)/libcfitsio_la-pliocomp.Plo \ +@@ -341,10 +354,13 @@ am__depfiles_remade = ./$(DEPDIR)/libcfitsio_la-buffers.Plo \ + ./$(DEPDIR)/libcfitsio_la-scalnull.Plo \ + ./$(DEPDIR)/libcfitsio_la-simplerng.Plo \ + ./$(DEPDIR)/libcfitsio_la-swapproc.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-trees.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-uncompr.Plo \ + ./$(DEPDIR)/libcfitsio_la-wcssub.Plo \ + ./$(DEPDIR)/libcfitsio_la-wcsutil.Plo \ + ./$(DEPDIR)/libcfitsio_la-zcompress.Plo \ + ./$(DEPDIR)/libcfitsio_la-zuncompress.Plo \ ++ ./$(DEPDIR)/libcfitsio_la-zutil.Plo \ + utilities/$(DEPDIR)/cookbook.Po \ + utilities/$(DEPDIR)/fitscopy.Po \ + utilities/$(DEPDIR)/fitsverify-ftverify.Po \ +@@ -616,7 +632,9 @@ libcfitsio_la_SOURCES = buffers.c cfileio.c checksum.c drvrfile.c \ + putkey.c region.c scalnull.c swapproc.c wcssub.c wcsutil.c \ + imcompress.c quantize.c ricecomp.c pliocomp.c fits_hcompress.c \ + fits_hdecompress.c simplerng.c zcompress.c zuncompress.c \ +- $(am__append_1) $(am__append_2) ++ adler32.c crc32.c deflate.c infback.c inffast.c inflate.c \ ++ inftrees.c trees.c uncompr.c zutil.c $(am__append_1) \ ++ $(am__append_2) + libcfitsio_la_CFLAGS = $(AM_CFLAGS) @DEFS@ + libcfitsio_swapproc_la_CFLAGS = $(libcfitsio_la_CFLAGS) @SSE_FLAGS@ + libcfitsio_la_LIBADD = -lm ${LIBS_CURL} ${LIBS} +@@ -878,9 +896,12 @@ mostlyclean-compile: + distclean-compile: + -rm -f *.tab.c + ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-adler32.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-buffers.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-cfileio.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-checksum.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-crc32.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-deflate.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-drvrfile.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-drvrgsiftp.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-drvrmem.Plo@am__quote@ # am--include-marker +@@ -916,6 +937,10 @@ distclean-compile: + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-grparser.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-histo.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-imcompress.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-infback.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-inffast.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-inflate.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-inftrees.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-iraffits.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-modkey.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-pliocomp.Plo@am__quote@ # am--include-marker +@@ -940,10 +965,13 @@ distclean-compile: + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-scalnull.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-simplerng.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-swapproc.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-trees.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-uncompr.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-wcssub.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-wcsutil.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-zcompress.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-zuncompress.Plo@am__quote@ # am--include-marker ++@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libcfitsio_la-zutil.Plo@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@utilities/$(DEPDIR)/cookbook.Po@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@utilities/$(DEPDIR)/fitscopy.Po@am__quote@ # am--include-marker + @AMDEP_TRUE@@am__include@ @am__quote@utilities/$(DEPDIR)/fitsverify-ftverify.Po@am__quote@ # am--include-marker +@@ -1417,6 +1445,76 @@ libcfitsio_la-zuncompress.lo: zuncompress.c + @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ + @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-zuncompress.lo `test -f 'zuncompress.c' || echo '$(srcdir)/'`zuncompress.c + ++libcfitsio_la-adler32.lo: adler32.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-adler32.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-adler32.Tpo -c -o libcfitsio_la-adler32.lo `test -f 'adler32.c' || echo '$(srcdir)/'`adler32.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-adler32.Tpo $(DEPDIR)/libcfitsio_la-adler32.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='adler32.c' object='libcfitsio_la-adler32.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-adler32.lo `test -f 'adler32.c' || echo '$(srcdir)/'`adler32.c ++ ++libcfitsio_la-crc32.lo: crc32.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-crc32.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-crc32.Tpo -c -o libcfitsio_la-crc32.lo `test -f 'crc32.c' || echo '$(srcdir)/'`crc32.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-crc32.Tpo $(DEPDIR)/libcfitsio_la-crc32.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='crc32.c' object='libcfitsio_la-crc32.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-crc32.lo `test -f 'crc32.c' || echo '$(srcdir)/'`crc32.c ++ ++libcfitsio_la-deflate.lo: deflate.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-deflate.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-deflate.Tpo -c -o libcfitsio_la-deflate.lo `test -f 'deflate.c' || echo '$(srcdir)/'`deflate.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-deflate.Tpo $(DEPDIR)/libcfitsio_la-deflate.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='deflate.c' object='libcfitsio_la-deflate.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-deflate.lo `test -f 'deflate.c' || echo '$(srcdir)/'`deflate.c ++ ++libcfitsio_la-infback.lo: infback.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-infback.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-infback.Tpo -c -o libcfitsio_la-infback.lo `test -f 'infback.c' || echo '$(srcdir)/'`infback.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-infback.Tpo $(DEPDIR)/libcfitsio_la-infback.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='infback.c' object='libcfitsio_la-infback.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-infback.lo `test -f 'infback.c' || echo '$(srcdir)/'`infback.c ++ ++libcfitsio_la-inffast.lo: inffast.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-inffast.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-inffast.Tpo -c -o libcfitsio_la-inffast.lo `test -f 'inffast.c' || echo '$(srcdir)/'`inffast.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-inffast.Tpo $(DEPDIR)/libcfitsio_la-inffast.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='inffast.c' object='libcfitsio_la-inffast.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-inffast.lo `test -f 'inffast.c' || echo '$(srcdir)/'`inffast.c ++ ++libcfitsio_la-inflate.lo: inflate.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-inflate.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-inflate.Tpo -c -o libcfitsio_la-inflate.lo `test -f 'inflate.c' || echo '$(srcdir)/'`inflate.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-inflate.Tpo $(DEPDIR)/libcfitsio_la-inflate.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='inflate.c' object='libcfitsio_la-inflate.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-inflate.lo `test -f 'inflate.c' || echo '$(srcdir)/'`inflate.c ++ ++libcfitsio_la-inftrees.lo: inftrees.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-inftrees.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-inftrees.Tpo -c -o libcfitsio_la-inftrees.lo `test -f 'inftrees.c' || echo '$(srcdir)/'`inftrees.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-inftrees.Tpo $(DEPDIR)/libcfitsio_la-inftrees.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='inftrees.c' object='libcfitsio_la-inftrees.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-inftrees.lo `test -f 'inftrees.c' || echo '$(srcdir)/'`inftrees.c ++ ++libcfitsio_la-trees.lo: trees.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-trees.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-trees.Tpo -c -o libcfitsio_la-trees.lo `test -f 'trees.c' || echo '$(srcdir)/'`trees.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-trees.Tpo $(DEPDIR)/libcfitsio_la-trees.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='trees.c' object='libcfitsio_la-trees.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-trees.lo `test -f 'trees.c' || echo '$(srcdir)/'`trees.c ++ ++libcfitsio_la-uncompr.lo: uncompr.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-uncompr.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-uncompr.Tpo -c -o libcfitsio_la-uncompr.lo `test -f 'uncompr.c' || echo '$(srcdir)/'`uncompr.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-uncompr.Tpo $(DEPDIR)/libcfitsio_la-uncompr.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='uncompr.c' object='libcfitsio_la-uncompr.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-uncompr.lo `test -f 'uncompr.c' || echo '$(srcdir)/'`uncompr.c ++ ++libcfitsio_la-zutil.lo: zutil.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-zutil.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-zutil.Tpo -c -o libcfitsio_la-zutil.lo `test -f 'zutil.c' || echo '$(srcdir)/'`zutil.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-zutil.Tpo $(DEPDIR)/libcfitsio_la-zutil.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='zutil.c' object='libcfitsio_la-zutil.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -c -o libcfitsio_la-zutil.lo `test -f 'zutil.c' || echo '$(srcdir)/'`zutil.c ++ + libcfitsio_la-f77_wrap1.lo: f77_wrap1.c + @am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libcfitsio_la_CFLAGS) $(CFLAGS) -MT libcfitsio_la-f77_wrap1.lo -MD -MP -MF $(DEPDIR)/libcfitsio_la-f77_wrap1.Tpo -c -o libcfitsio_la-f77_wrap1.lo `test -f 'f77_wrap1.c' || echo '$(srcdir)/'`f77_wrap1.c + @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libcfitsio_la-f77_wrap1.Tpo $(DEPDIR)/libcfitsio_la-f77_wrap1.Plo +@@ -1875,9 +1973,12 @@ clean-am: clean-binPROGRAMS clean-generic clean-libLTLIBRARIES \ + + distclean: distclean-am + -rm -f $(am__CONFIG_DISTCLEAN_FILES) ++ -rm -f ./$(DEPDIR)/libcfitsio_la-adler32.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-buffers.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-cfileio.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-checksum.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-crc32.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-deflate.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-drvrfile.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-drvrgsiftp.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-drvrmem.Plo +@@ -1913,6 +2014,10 @@ distclean: distclean-am + -rm -f ./$(DEPDIR)/libcfitsio_la-grparser.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-histo.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-imcompress.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-infback.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-inffast.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-inflate.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-inftrees.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-iraffits.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-modkey.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-pliocomp.Plo +@@ -1937,10 +2042,13 @@ distclean: distclean-am + -rm -f ./$(DEPDIR)/libcfitsio_la-scalnull.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-simplerng.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-swapproc.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-trees.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-uncompr.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-wcssub.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-wcsutil.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-zcompress.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-zuncompress.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-zutil.Plo + -rm -f utilities/$(DEPDIR)/cookbook.Po + -rm -f utilities/$(DEPDIR)/fitscopy.Po + -rm -f utilities/$(DEPDIR)/fitsverify-ftverify.Po +@@ -2004,9 +2112,12 @@ installcheck-am: + maintainer-clean: maintainer-clean-am + -rm -f $(am__CONFIG_DISTCLEAN_FILES) + -rm -rf $(top_srcdir)/autom4te.cache ++ -rm -f ./$(DEPDIR)/libcfitsio_la-adler32.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-buffers.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-cfileio.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-checksum.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-crc32.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-deflate.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-drvrfile.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-drvrgsiftp.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-drvrmem.Plo +@@ -2042,6 +2153,10 @@ maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/libcfitsio_la-grparser.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-histo.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-imcompress.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-infback.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-inffast.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-inflate.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-inftrees.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-iraffits.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-modkey.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-pliocomp.Plo +@@ -2066,10 +2181,13 @@ maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/libcfitsio_la-scalnull.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-simplerng.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-swapproc.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-trees.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-uncompr.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-wcssub.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-wcsutil.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-zcompress.Plo + -rm -f ./$(DEPDIR)/libcfitsio_la-zuncompress.Plo ++ -rm -f ./$(DEPDIR)/libcfitsio_la-zutil.Plo + -rm -f utilities/$(DEPDIR)/cookbook.Po + -rm -f utilities/$(DEPDIR)/fitscopy.Po + -rm -f utilities/$(DEPDIR)/fitsverify-ftverify.Po diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000..fc4c1d2 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,26 @@ +# Patches for cfitsio + +This directory contains patches for the cfitsio build. These patches +are applied before the library is compiled during the python package +build step. + +The patches were generated with the script `build_cfitsio_patches.py` by +Matthew Becker in December of 2018. + +## Adding New Patches + +To add new patches, you need to + +1. Make a copy of the file you want to patch. +2. Modify it. +3. Call `diff -u old_file new_file` to a get a unified format patch. +4. Make sure the paths in the patch at the top look like this + ``` + --- cfitsio/ 2018-03-01 10:28:51.000000000 -0600 + +++ cfitsio/ 2018-12-14 08:39:20.000000000 -0600 + ... + ``` + where `` and `` have a cfitsio version and + file that is being patched. + +5. Commit the patch file in the patches directory with the name `.patch`. diff --git a/patches/build_cfitsio_patches.py b/patches/build_cfitsio_patches.py new file mode 100644 index 0000000..5d7889c --- /dev/null +++ b/patches/build_cfitsio_patches.py @@ -0,0 +1,35 @@ +import os +import argparse + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--moddir', required=True, + help='directory containing modified files') + parser.add_argument('--dir', required=True, + help='directory containing unmodified files') + parser.add_argument('--patch-dir', required=True) + return parser.parse_args() + + +def main(): + args = get_args() + + os.makedirs(args.patch_dir, exist_ok=True) + + for root, _, files in os.walk(args.dir): + for fname in files: + src = os.path.join(args.dir, fname) + dst = os.path.join(args.moddir, fname) + patch = os.path.join(args.patch_dir, fname + '.patch') + os.system('diff -u %s %s > %s' % (src, dst, patch)) + with open(patch, 'rb') as fp: + buff = fp.read() + if len(buff) == 0: + os.remove(patch) + else: + print(fname) + break + + +main() diff --git a/patches/configure.ac.patch b/patches/configure.ac.patch new file mode 100644 index 0000000..75f6900 --- /dev/null +++ b/patches/configure.ac.patch @@ -0,0 +1,21 @@ +--- cfitsio-4.2.0/configure.ac 2022-10-31 14:40:23.000000000 -0400 ++++ cfitsio-4.2.0/configure.ac 2023-07-14 11:45:00.797390794 -0400 +@@ -170,11 +170,16 @@ AC_ARG_WITH( + [AS_HELP_STRING([--with-bzip2[[=PATH]]],[Enable bzip2 support. Optional path to the location of include/bzlib.h and lib/libbz2])], + [ if test "x$withval" != "xno"; then + if test "x$withval" = "xyes" ; then +- AC_CHECK_LIB([bz2],[main],[],[AC_MSG_ERROR(Unable to locate bz2 library needed when enabling bzip2 support; try specifying the path)]) ++ AC_CHECK_LIB( ++ [bz2], ++ [main], ++ [LIBS="$LIBS -lbz2"; AC_DEFINE(HAVE_LIBBZ2) AC_CHECK_HEADERS(bzlib.h,[AC_DEFINE(HAVE_BZIP2,1,[Define if you want bzip2 read support])])], ++ [] ++ ) + else + BZIP2_PATH="${withval}" ++ AC_CHECK_HEADERS(bzlib.h,[AC_DEFINE(HAVE_BZIP2,1,[Define if you want bzip2 read support])]) + fi +- AC_CHECK_HEADERS(bzlib.h,[AC_DEFINE(HAVE_BZIP2,1,[Define if you want bzip2 read support])]) + fi + ] + ) diff --git a/patches/configure.patch b/patches/configure.patch new file mode 100644 index 0000000..01c5372 --- /dev/null +++ b/patches/configure.patch @@ -0,0 +1,42 @@ +--- cfitsio-4.2.0/configure 2022-10-31 14:40:23.000000000 -0400 ++++ cfitsio-4.2.0/configure 2023-07-14 11:46:53.298055665 -0400 +@@ -15493,19 +15493,24 @@ fi + printf "%s\n" "$ac_cv_lib_bz2_main" >&6; } + if test "x$ac_cv_lib_bz2_main" = xyes + then : +- printf "%s\n" "#define HAVE_LIBBZ2 1" >>confdefs.h ++ LIBS="$LIBS -lbz2"; printf "%s\n" "#define HAVE_LIBBZ2 1" >>confdefs.h ++ for ac_header in bzlib.h ++do : ++ ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" ++if test "x$ac_cv_header_bzlib_h" = xyes ++then : ++ printf "%s\n" "#define HAVE_BZLIB_H 1" >>confdefs.h + +- LIBS="-lbz2 $LIBS" ++printf "%s\n" "#define HAVE_BZIP2 1" >>confdefs.h + +-else case e in #( +- e) as_fn_error $? "Unable to locate bz2 library needed when enabling bzip2 support; try specifying the path" "$LINENO" 5 ;; +-esac ++fi ++ ++done + fi + + else + BZIP2_PATH="${withval}" +- fi +- for ac_header in bzlib.h ++ for ac_header in bzlib.h + do : + ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default" + if test "x$ac_cv_header_bzlib_h" = xyes +@@ -15517,6 +15522,7 @@ printf "%s\n" "#define HAVE_BZIP2 1" >>confdefs.h + fi + + done ++ fi + fi + + diff --git a/patches/fitsio2.h.patch b/patches/fitsio2.h.patch new file mode 100644 index 0000000..f56d7c6 --- /dev/null +++ b/patches/fitsio2.h.patch @@ -0,0 +1,21 @@ +--- cfitsio-4.2.0/fitsio2.h 2023-08-09 10:23:45.508392645 +0800 ++++ cfitsio-4.2.0/fitsio2.h 2023-08-09 10:29:44.960511085 +0800 +@@ -151,6 +151,18 @@ + # error "can't handle long size given by __riscv_xlen" + # endif + ++#elif defined(__loongarch__) ++ ++#define BYTESWAPPED TRUE ++ ++# if __loongarch_grlen == 32 ++# define LONGSIZE 32 ++# elif __loongarch_grlen == 64 ++# define LONGSIZE 64 ++# else ++# error "can't handle long size given by __loongarch_grlen" ++# endif ++ + /* ============================================================== */ + /* the following are all 32-bit byteswapped platforms */ + diff --git a/patches/getcold.c.patch b/patches/getcold.c.patch new file mode 100644 index 0000000..c55918c --- /dev/null +++ b/patches/getcold.c.patch @@ -0,0 +1,38 @@ +--- a/getcold.c ++++ b/getcold.c +@@ -1418,7 +1418,7 @@ int fffr4r8(float *input, /* I - array of values to be converted */ + nullarray[ii] = 1; + } + else /* it's an underflow */ +- output[ii] = 0; ++ output[ii] = (double) input[ii]; + } + else + output[ii] = (double) input[ii]; +@@ -1439,7 +1439,7 @@ int fffr4r8(float *input, /* I - array of values to be converted */ + nullarray[ii] = 1; + } + else /* it's an underflow */ +- output[ii] = zero; ++ output[ii] = input[ii] * scale + zero; + } + else + output[ii] = input[ii] * scale + zero; +@@ -1519,7 +1519,7 @@ int fffr8r8(double *input, /* I - array of values to be converted */ + } + } + else /* it's an underflow */ +- output[ii] = 0; ++ output[ii] = input[ii]; + } + else + output[ii] = input[ii]; +@@ -1544,7 +1544,7 @@ int fffr8r8(double *input, /* I - array of values to be converted */ + } + } + else /* it's an underflow */ +- output[ii] = zero; ++ output[ii] = input[ii] * scale + zero; + } + else + output[ii] = input[ii] * scale + zero; diff --git a/patches/getcole.c.patch b/patches/getcole.c.patch new file mode 100644 index 0000000..bc850e3 --- /dev/null +++ b/patches/getcole.c.patch @@ -0,0 +1,40 @@ +--- a/getcole.c ++++ b/getcole.c +@@ -1425,7 +1425,7 @@ int fffr4r4(float *input, /* I - array of values to be converted */ + } + } + else /* it's an underflow */ +- output[ii] = 0; ++ output[ii] = input[ii]; + } + else + output[ii] = input[ii]; +@@ -1450,7 +1450,7 @@ int fffr4r4(float *input, /* I - array of values to be converted */ + } + } + else /* it's an underflow */ +- output[ii] = (float) zero; ++ output[ii] = (float) (input[ii] * scale + zero); + } + else + output[ii] = (float) (input[ii] * scale + zero); +@@ -1549,8 +1549,8 @@ int fffr8r4(double *input, /* I - array of values to be converted */ + else + nullarray[ii] = 1; + } +- else /* it's an underflow */ +- output[ii] = 0; ++ else ++ output[ii] = (float) input[ii]; + } + else + { +@@ -1596,7 +1596,7 @@ int fffr8r4(double *input, /* I - array of values to be converted */ + output[ii] = FLT_MAX; + } + else +- output[ii] = (float) zero; ++ output[ii] = (float) (input[ii] * scale + zero); + } + } + else diff --git a/patches/imcompress.c.patch b/patches/imcompress.c.patch new file mode 100644 index 0000000..65a173f --- /dev/null +++ b/patches/imcompress.c.patch @@ -0,0 +1,213 @@ +--- a/imcompress.c ++++ b/imcompress.c +@@ -2217,6 +2217,17 @@ int imcomp_compress_tile (fitsfile *outfptr, + ffpclb(outfptr, (outfptr->Fptr)->cn_gzip_data, row, 1, + gzip_nelem, (unsigned char *) cbuf, status); + ++ /* we must zero out existing compressed data if it exists. */ ++ /* otherwise on read this data is read ahead of the gzipped */ ++ /* data and will cause a bug. */ ++ LONGLONG _test_nelemll, _test_offset; ++ ffgdesll(outfptr, (outfptr->Fptr)->cn_compressed, row, &_test_nelemll, &_test_offset, ++ status); ++ if (_test_nelemll) { ++ ffpclb(outfptr, (outfptr->Fptr)->cn_compressed, row, 1, ++ 0, NULL, status); ++ } ++ + free(cbuf); /* finished with this buffer */ + } + +@@ -6453,20 +6453,7 @@ int imcomp_decompress_tile (fitsfile *infptr, + { + pixlen = sizeof(short); + +- if ((infptr->Fptr)->quantize_level == NO_QUANTIZE) { +- /* the floating point pixels were losselessly compressed with GZIP */ +- /* Just have to copy the values to the output array */ +- +- if (tiledatatype == TINT) { +- fffr4i2((float *) idata, tilelen, bscale, bzero, nullcheck, +- *(short *) nulval, bnullarray, anynul, +- (short *) buffer, status); +- } else { +- fffr8i2((double *) idata, tilelen, bscale, bzero, nullcheck, +- *(short *) nulval, bnullarray, anynul, +- (short *) buffer, status); +- } +- } else if (tiledatatype == TINT) { ++ if (tiledatatype == TINT) { + if ((infptr->Fptr)->compress_type == PLIO_1 && actual_bzero == 32768.) { + /* special case where unsigned 16-bit integers have been */ + /* offset by +32768 when using PLIO */ +@@ -6500,26 +6487,17 @@ int imcomp_decompress_tile (fitsfile *infptr, + fffi1i2((unsigned char *)idata, tilelen, bscale, bzero, nullcheck, (unsigned char) tnull, + *(short *) nulval, bnullarray, anynul, + (short *) buffer, status); ++ } else { ++ fffi8i2((LONGLONG *) idata, tilelen, bscale, bzero, nullcheck, tnull, ++ *(short *) nulval, bnullarray, anynul, ++ (short *) buffer, status); + } + } + else if (datatype == TINT) + { + pixlen = sizeof(int); + +- if ((infptr->Fptr)->quantize_level == NO_QUANTIZE) { +- /* the floating point pixels were losselessly compressed with GZIP */ +- /* Just have to copy the values to the output array */ +- +- if (tiledatatype == TINT) { +- fffr4int((float *) idata, tilelen, bscale, bzero, nullcheck, +- *(int *) nulval, bnullarray, anynul, +- (int *) buffer, status); +- } else { +- fffr8int((double *) idata, tilelen, bscale, bzero, nullcheck, +- *(int *) nulval, bnullarray, anynul, +- (int *) buffer, status); +- } +- } else if (tiledatatype == TINT) ++ if (tiledatatype == TINT) + if ((infptr->Fptr)->compress_type == PLIO_1 && actual_bzero == 32768.) { + /* special case where unsigned 16-bit integers have been */ + /* offset by +32768 when using PLIO */ +@@ -6539,25 +6517,16 @@ int imcomp_decompress_tile (fitsfile *infptr, + fffi1int((unsigned char *)idata, tilelen, bscale, bzero, nullcheck, (unsigned char) tnull, + *(int *) nulval, bnullarray, anynul, + (int *) buffer, status); ++ else ++ fffi8int((LONGLONG *) idata, (long) tilelen, bscale, bzero, nullcheck, tnull, ++ *(int *) nulval, bnullarray, anynul, ++ (int *) buffer, status); + } + else if (datatype == TLONG) + { + pixlen = sizeof(long); + +- if ((infptr->Fptr)->quantize_level == NO_QUANTIZE) { +- /* the floating point pixels were losselessly compressed with GZIP */ +- /* Just have to copy the values to the output array */ +- +- if (tiledatatype == TINT) { +- fffr4i4((float *) idata, tilelen, bscale, bzero, nullcheck, +- *(long *) nulval, bnullarray, anynul, +- (long *) buffer, status); +- } else { +- fffr8i4((double *) idata, tilelen, bscale, bzero, nullcheck, +- *(long *) nulval, bnullarray, anynul, +- (long *) buffer, status); +- } +- } else if (tiledatatype == TINT) ++ if (tiledatatype == TINT) + if ((infptr->Fptr)->compress_type == PLIO_1 && actual_bzero == 32768.) { + /* special case where unsigned 16-bit integers have been */ + /* offset by +32768 when using PLIO */ +@@ -6577,6 +6546,11 @@ int imcomp_decompress_tile (fitsfile *infptr, + fffi1i4((unsigned char *)idata, tilelen, bscale, bzero, nullcheck, (unsigned char) tnull, + *(long *) nulval, bnullarray, anynul, + (long *) buffer, status); ++ else ++ fffi8i4((LONGLONG *) idata, tilelen, bscale, bzero, nullcheck, tnull, ++ *(long *) nulval, bnullarray, anynul, ++ (long *) buffer, status); ++ + } + else if (datatype == TFLOAT) + { +@@ -6745,20 +6719,7 @@ int imcomp_decompress_tile (fitsfile *infptr, + { + pixlen = sizeof(short); + +- if ((infptr->Fptr)->quantize_level == NO_QUANTIZE) { +- /* the floating point pixels were losselessly compressed with GZIP */ +- /* Just have to copy the values to the output array */ +- +- if (tiledatatype == TINT) { +- fffr4u2((float *) idata, tilelen, bscale, bzero, nullcheck, +- *(unsigned short *) nulval, bnullarray, anynul, +- (unsigned short *) buffer, status); +- } else { +- fffr8u2((double *) idata, tilelen, bscale, bzero, nullcheck, +- *(unsigned short *) nulval, bnullarray, anynul, +- (unsigned short *) buffer, status); +- } +- } else if (tiledatatype == TINT) ++ if (tiledatatype == TINT) + if ((infptr->Fptr)->compress_type == PLIO_1 && actual_bzero == 32768.) { + /* special case where unsigned 16-bit integers have been */ + /* offset by +32768 when using PLIO */ +@@ -6778,26 +6739,16 @@ int imcomp_decompress_tile (fitsfile *infptr, + fffi1u2((unsigned char *)idata, tilelen, bscale, bzero, nullcheck, (unsigned char) tnull, + *(unsigned short *) nulval, bnullarray, anynul, + (unsigned short *) buffer, status); ++ else ++ fffi8u2((LONGLONG *) idata, tilelen, bscale, bzero, nullcheck, tnull, ++ *(unsigned short *) nulval, bnullarray, anynul, ++ (unsigned short *) buffer, status); + } + else if (datatype == TUINT) + { + pixlen = sizeof(int); + +- if ((infptr->Fptr)->quantize_level == NO_QUANTIZE) { +- /* the floating point pixels were losselessly compressed with GZIP */ +- /* Just have to copy the values to the output array */ +- +- if (tiledatatype == TINT) { +- fffr4uint((float *) idata, tilelen, bscale, bzero, nullcheck, +- *(unsigned int *) nulval, bnullarray, anynul, +- (unsigned int *) buffer, status); +- } else { +- fffr8uint((double *) idata, tilelen, bscale, bzero, nullcheck, +- *(unsigned int *) nulval, bnullarray, anynul, +- (unsigned int *) buffer, status); +- } +- } else +- if (tiledatatype == TINT) ++ if (tiledatatype == TINT) + if ((infptr->Fptr)->compress_type == PLIO_1 && actual_bzero == 32768.) { + /* special case where unsigned 16-bit integers have been */ + /* offset by +32768 when using PLIO */ +@@ -6817,25 +6768,16 @@ int imcomp_decompress_tile (fitsfile *infptr, + fffi1uint((unsigned char *)idata, tilelen, bscale, bzero, nullcheck, (unsigned char) tnull, + *(unsigned int *) nulval, bnullarray, anynul, + (unsigned int *) buffer, status); ++ else ++ fffi8uint((LONGLONG *) idata, tilelen, bscale, bzero, nullcheck, tnull, ++ *(unsigned int *) nulval, bnullarray, anynul, ++ (unsigned int *) buffer, status); + } + else if (datatype == TULONG) + { + pixlen = sizeof(long); + +- if ((infptr->Fptr)->quantize_level == NO_QUANTIZE) { +- /* the floating point pixels were losselessly compressed with GZIP */ +- /* Just have to copy the values to the output array */ +- +- if (tiledatatype == TINT) { +- fffr4u4((float *) idata, tilelen, bscale, bzero, nullcheck, +- *(unsigned long *) nulval, bnullarray, anynul, +- (unsigned long *) buffer, status); +- } else { +- fffr8u4((double *) idata, tilelen, bscale, bzero, nullcheck, +- *(unsigned long *) nulval, bnullarray, anynul, +- (unsigned long *) buffer, status); +- } +- } else if (tiledatatype == TINT) ++ if (tiledatatype == TINT) + if ((infptr->Fptr)->compress_type == PLIO_1 && actual_bzero == 32768.) { + /* special case where unsigned 16-bit integers have been */ + /* offset by +32768 when using PLIO */ +@@ -6855,6 +6797,10 @@ int imcomp_decompress_tile (fitsfile *infptr, + fffi1u4((unsigned char *)idata, tilelen, bscale, bzero, nullcheck, (unsigned char) tnull, + *(unsigned long *) nulval, bnullarray, anynul, + (unsigned long *) buffer, status); ++ else ++ fffi8u4((LONGLONG *) idata, tilelen, bscale, bzero, nullcheck, tnull, ++ *(unsigned long *) nulval, bnullarray, anynul, ++ (unsigned long *) buffer, status); + } + else + *status = BAD_DATATYPE; diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..444b782 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,17 @@ +exclude = [ + ".git", + "build", + "dist", + "cfitsio-*", + "patches", + "zlib", + ".github", +] +line-length = 79 + +[lint] +select = ["E", "F", "W"] +preview = true + +[format] +quote-style = "preserve" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d185146 --- /dev/null +++ b/setup.py @@ -0,0 +1,485 @@ +# +# setup script for fitsio, using setuptools +# +# c.f. +# https://packaging.python.org/guides/distributing-packages-using-setuptools/ + +from __future__ import print_function +from setuptools import setup, Extension, find_packages +from setuptools.command.build_ext import build_ext + +import warnings +import tempfile +import tarfile +import sys +import os +import subprocess +from subprocess import PIPE +import glob +import shutil + +if "FITSIO_FAIL_ON_BAD_PATCHES" in os.environ: + if os.environ["FITSIO_FAIL_ON_BAD_PATCHES"].lower() in ["false", "0"]: + FITSIO_FAIL_ON_BAD_PATCHES = False + else: + FITSIO_FAIL_ON_BAD_PATCHES = True +else: + FITSIO_FAIL_ON_BAD_PATCHES = True + +if "--use-system-fitsio" in sys.argv: + del sys.argv[sys.argv.index("--use-system-fitsio")] + USE_SYSTEM_FITSIO = True +else: + USE_SYSTEM_FITSIO = False or "FITSIO_USE_SYSTEM_FITSIO" in os.environ + +if "--system-fitsio-includedir" in sys.argv or any( + a.startswith("--system-fitsio-includedir=") for a in sys.argv +): + if "--system-fitsio-includedir" in sys.argv: + ind = sys.argv.index("--system-fitsio-includedir") + SYSTEM_FITSIO_INCLUDEDIR = sys.argv[ind + 1] + del sys.argv[ind + 1] + del sys.argv[ind] + else: + for ind in range(len(sys.argv)): + if sys.argv[ind].startswith("--system-fitsio-includedir="): + break + SYSTEM_FITSIO_INCLUDEDIR = sys.argv[ind].split("=", 1)[1] + del sys.argv[ind] +else: + SYSTEM_FITSIO_INCLUDEDIR = os.environ.get( + "FITSIO_SYSTEM_FITSIO_INCLUDEDIR", + None, + ) + + +if "--system-fitsio-libdir" in sys.argv or any( + a.startswith("--system-fitsio-libdir=") for a in sys.argv +): + if "--system-fitsio-libdir" in sys.argv: + ind = sys.argv.index("--system-fitsio-libdir") + SYSTEM_FITSIO_LIBDIR = sys.argv[ind + 1] + del sys.argv[ind + 1] + del sys.argv[ind] + else: + for ind in range(len(sys.argv)): + if sys.argv[ind].startswith("--system-fitsio-libdir="): + break + SYSTEM_FITSIO_LIBDIR = sys.argv[ind].split("=", 1)[1] + del sys.argv[ind] +else: + SYSTEM_FITSIO_LIBDIR = os.environ.get( + "FITSIO_SYSTEM_FITSIO_LIBDIR", + None, + ) + + +def _print_msg(text): + print("\n" + "=" * 79 + f"\n{text}\n" + "=" * 79, flush=True) + + +class build_ext_subclass(build_ext): + cfitsio_version = '4.6.3' + cfitsio_dir = 'cfitsio-%s' % cfitsio_version + + def finalize_options(self): + build_ext.finalize_options(self) + + self.cfitsio_build_dir = os.path.join( + self.build_temp, self.cfitsio_dir + ) + self.cfitsio_zlib_dir = os.path.join(self.cfitsio_build_dir, 'zlib') + self.cfitsio_patch_dir = os.path.join(self.build_temp, 'patches') + + if USE_SYSTEM_FITSIO: + if SYSTEM_FITSIO_INCLUDEDIR is not None: + self.include_dirs.insert(0, SYSTEM_FITSIO_INCLUDEDIR) + if SYSTEM_FITSIO_LIBDIR is not None: + self.library_dirs.insert(0, SYSTEM_FITSIO_LIBDIR) + else: + # We defer configuration of the bundled cfitsio to build_extensions + # because we will know the compiler there. + self.include_dirs.insert(0, self.cfitsio_build_dir) + + def run(self): + # For extensions that require 'numpy' in their include dirs, + # replace 'numpy' with the actual paths + import numpy + + np_include = numpy.get_include() + + for extension in self.extensions: + if 'numpy' in extension.include_dirs: + idx = extension.include_dirs.index('numpy') + extension.include_dirs.insert(idx, np_include) + extension.include_dirs.remove('numpy') + + build_ext.run(self) + + def build_extensions(self): + if not USE_SYSTEM_FITSIO: + # Use the compiler for building python to build cfitsio + # for maximized compatibility. + + # turns out we need to set the include dirs here too + # directly for the compiler + self.compiler.include_dirs.insert(0, self.cfitsio_build_dir) + + CCold = self.compiler.compiler + if 'ccache' in CCold: + CC = [] + for val in CCold: + if val == 'ccache': + _print_msg("removing ccache from the compiler options") + continue + + CC.append(val) + else: + CC = None + + self.configure_cfitsio( + CC=CC, + ARCHIVE=self.compiler.archiver, + RANLIB=self.compiler.ranlib, + ) + + # If configure detected bzlib.h, we have to link to libbz2 + with open(os.path.join(self.cfitsio_build_dir, 'Makefile')) as fp: + _makefile = fp.read() + _have_bzip2 = False + _have_curl = False + for line in _makefile.splitlines(): + for _part in line.split("="): + for _eqpart in _part.split(): + if "-lbz2" in _eqpart: + _have_bzip2 = True + if "-lcurl" in _eqpart: + _have_curl = True + if _have_bzip2: + _print_msg( + "found -lbz2 in Makefile\n" + "linking Python extension to bzip2" + ) + self.compiler.add_library('bz2') + self.compiler.define_macro('FITSIO_HAS_BZIP2_SUPPORT') + else: + _print_msg( + "did not find -lbz2 in Makefile\n" + "bzip2 support is disabled" + ) + + if _have_curl: + _print_msg( + "found -lcurl in Makefile\n" + "linking Python extension to curl" + ) + self.compiler.add_library('curl') + self.compiler.define_macro('FITSIO_HAS_CURL_SUPPORT') + else: + _print_msg( + "did not find -lcurl in Makefile\n" + "curl support is disabled" + ) + + self.compile_cfitsio() + + # link against the .a library in cfitsio; + # It should have been a 'static' library of relocatable objects + # (-fPIC), since we use the python compiler flags + + link_objects = glob.glob( + os.path.join(self.cfitsio_build_dir, '*.o') + ) + + self.compiler.set_link_objects(link_objects) + + # Ultimate hack: append the .a files to the dependency list + # so they will be properly rebuild if cfitsio source is updated. + for ext in self.extensions: + ext.depends += link_objects + else: + self.compiler.add_library('cfitsio') + + # Check if system cfitsio was compiled with bzip2 and/or curl + if self.check_system_cfitsio_objects('bzip2'): + _print_msg( + "found bz2 symbol in system cfitsio library\n" + "linking Python extension to bzip2" + ) + self.compiler.add_library('bz2') + self.compiler.define_macro('FITSIO_HAS_BZIP2_SUPPORT') + else: + _print_msg( + "did not find bz2 symbol in system cfitsio library\n" + "bzip2 support is disabled" + ) + + if self.check_system_cfitsio_objects('curl_'): + _print_msg( + "found curl_ symbol in system cfitsio library\n" + "linking Python extension to curl" + ) + self.compiler.add_library('curl') + self.compiler.define_macro('FITSIO_HAS_CURL_SUPPORT') + else: + _print_msg( + "did not find curl_ symbol in system cfitsio library\n" + "curl support is disabled" + ) + + self.compiler.define_macro('FITSIO_USING_SYSTEM_FITSIO') + + self.compiler.add_library('z') + + # fitsio requires libm as well. + self.compiler.add_library('m') + + # call the original build_extensions + build_ext.build_extensions(self) + + def patch_cfitsio(self): + _print_msg("patching cfitsio") + + try: + subprocess.check_call(["patch", "-v"]) + except subprocess.CalledProcessError as e: + warnings.warn( + "`patch` command not found! " + "Some bugs in cfitsio may not be fixed! " + "See the patches we carry at " + "https://github.com/esheldon/fitsio/tree/master/patches." + ) + if FITSIO_FAIL_ON_BAD_PATCHES: + raise e + else: + return + + patches = glob.glob('%s/*.patch' % self.cfitsio_patch_dir) + for patch in patches: + fname = os.path.basename(patch.replace('.patch', '')) + try: + subprocess.check_call( + [ + "patch", + "-N", + "--dry-run", + "%s/%s" % (self.cfitsio_build_dir, fname), + patch, + ] + ) + except subprocess.CalledProcessError as e: + warnings.warn( + "Failed to apply patch: " + os.path.basename(patch) + ) + if FITSIO_FAIL_ON_BAD_PATCHES: + raise e + else: + subprocess.check_call( + [ + "patch", + "%s/%s" % (self.cfitsio_build_dir, fname), + patch, + ], + ) + + def configure_cfitsio(self, CC=None, ARCHIVE=None, RANLIB=None): + # prepare source code and run configure + def copy_update(dir1, dir2): + f1 = os.listdir(dir1) + for f in f1: + path1 = os.path.join(dir1, f) + path2 = os.path.join(dir2, f) + + if os.path.isdir(path1): + if not os.path.exists(path2): + os.makedirs(path2) + copy_update(path1, path2) + else: + if not os.path.exists(path2): + shutil.copy(path1, path2) + else: + stat1 = os.stat(path1) + stat2 = os.stat(path2) + if stat1.st_mtime > stat2.st_mtime: + shutil.copy(path1, path2) + + if not os.path.exists('build'): + os.makedirs('build') + + if not os.path.exists(self.cfitsio_build_dir): + os.makedirs(self.cfitsio_build_dir) + + if not os.path.exists(self.cfitsio_patch_dir): + os.makedirs(self.cfitsio_patch_dir) + + if sys.version_info.major >= 3 and sys.version_info.minor >= 12: + tar_kwargs = {"filter": "fully_trusted"} + else: + tar_kwargs = {} + + with tempfile.TemporaryDirectory() as tmpdir: + if os.path.exists(self.cfitsio_dir) and os.path.isdir( + self.cfitsio_dir + ): + _print_msg( + "using cfitsio source code from " + f"{self.cfitsio_dir} for debugging" + ) + copy_update( + self.cfitsio_dir, + self.cfitsio_build_dir, + ) + else: + with tarfile.open(self.cfitsio_dir + ".tar.gz", "r:gz") as tar: + tar.extractall(path=tmpdir, **tar_kwargs) + copy_update( + os.path.join(tmpdir, self.cfitsio_dir), + self.cfitsio_build_dir, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + with tarfile.open("zlib.tar.gz", "r:gz") as tar: + tar.extractall(path=tmpdir, **tar_kwargs) + copy_update( + os.path.join(tmpdir, "zlib"), self.cfitsio_build_dir + ) + + copy_update('patches', self.cfitsio_patch_dir) + + # we patch the source in the buil dir to avoid mucking with the repo + self.patch_cfitsio() + + makefile = os.path.join(self.cfitsio_build_dir, 'Makefile') + + if os.path.exists(makefile): + # Makefile already there + _print_msg("found Makefile so not running configure!") + return + else: + _print_msg("configuring cfitsio") + + # the latest cfitsio build system links its example + # programs (e.g., `cookbook`) against the shared library. + # when we use `-fvisibility=hidden` in the CFLAGS ( + # needed to hide the cfitsio symbols in the python `.so``), + # the linking against the shared library fails. + # so we disable shared libraries with (`--disable-shared``) + # and add `-fPIC` to the flags to ensure the python `.so` + # works properly later + args = [ + '--without-fortran', + '--disable-shared', + ] + our_cflags = "-fPIC -fvisibility=hidden" + + if "FITSIO_BZIP2_DIR" in os.environ: + if not os.environ["FITSIO_BZIP2_DIR"]: + args += ["--with-bzip2"] + else: + args += ['--with-bzip2="%s"' % os.environ["FITSIO_BZIP2_DIR"]] + else: + # let autoconf detect if we have bzip2 + args += ['--with-bzip2'] + + env = {} + env.update(os.environ) + + if CC is not None: + env["CC"] = ' '.join(CC[:1]) + env["CFLAGS"] = ' '.join(CC[1:]) + our_cflags + else: + if "CFLAGS" in os.environ: + env["CFLAGS"] = os.environ["CFLAGS"] + " " + our_cflags + else: + env["CFLAGS"] = our_cflags + + if ARCHIVE: + env["ARCHIVE"] = ' '.join(ARCHIVE) + if RANLIB: + env["RANLIB"] = ' '.join(RANLIB) + + res = subprocess.run( + ["sh", "./configure"] + args, + cwd=self.cfitsio_build_dir, + env=env, + ) + if res.returncode != 0: + with open( + os.path.join(self.cfitsio_build_dir, "config.log") + ) as fp: + logfile = fp.read() + raise ValueError( + "could not configure cfitsio %s: config.log:\n\n%s" + % ( + self.cfitsio_version, + logfile, + ) + ) + + def compile_cfitsio(self): + _print_msg("building cfitsio") + res = subprocess.run( + "make", + cwd=self.cfitsio_build_dir, + ) + if res.returncode != 0: + raise ValueError( + "could not compile cfitsio %s" % self.cfitsio_version + ) + + def check_system_cfitsio_objects(self, obj_name): + for lib_dir in self.library_dirs: + if os.path.isfile('%s/libcfitsio.a' % (lib_dir)): + res = subprocess.run( + ["nm", "-g", "%s/libcfitsio.a" % lib_dir], + stdout=PIPE, + stderr=PIPE, + ) + for line in res.stdout.decode("utf-8").splitlines(): + if obj_name in line: + return True + + return False + return False + + +sources = ["fitsio/fitsio_pywrap.c"] + +ext = Extension("fitsio._fitsio_wrap", sources, include_dirs=['numpy']) + +description = ( + "A full featured python library to read from and write to FITS files." +) + +with open(os.path.join(os.path.dirname(__file__), "README.md")) as fp: + long_description = fp.read() + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: GNU General Public License (GPL)", + "Topic :: Scientific/Engineering :: Astronomy", + "Intended Audience :: Science/Research", +] + +setup( + name="fitsio", + description=description, + long_description=long_description, + long_description_content_type='text/markdown; charset=UTF-8; variant=GFM', + license="GPL", + classifiers=classifiers, + url="https://github.com/esheldon/fitsio", + author="Erin Scott Sheldon", + author_email="erin.sheldon@gmail.com", + setup_requires=['numpy>=1.7', 'setuptools-scm>=8'], + install_requires=['numpy>=1.7'], + packages=find_packages(), + python_requires=">=3.8", + include_package_data=True, + ext_modules=[ext], + cmdclass={"build_ext": build_ext_subclass}, + use_scm_version={ + "version_file": "fitsio/_version.py", + "version_file_template": "__version__ = '{version}'\n", + }, +) diff --git a/zlib.tar.gz b/zlib.tar.gz new file mode 100644 index 0000000..f4b196b Binary files /dev/null and b/zlib.tar.gz differ