Implement Custom Types -- AttributeType

.devcontainer/Dockerfile

@@ -8,6 +8,7 @@ RUN \
     pip uninstall datajoint -y
 
 USER root
-ENV DJ_HOST db
-ENV DJ_USER root
-ENV DJ_PASS password
+ENV DJ_HOST=db
+ENV DJ_USER=root
+ENV DJ_PASS=password
+ENV S3_ENDPOINT=minio:9000

.devcontainer/devcontainer.json

@@ -1,7 +1,6 @@
 {
-  "image": "mcr.microsoft.com/devcontainers/typescript-node:0-18",
-  "features": {
-    "ghcr.io/devcontainers/features/docker-in-docker:2": {}
-  },
+  "dockerComposeFile": ["../docker-compose.yaml", "docker-compose.yml"],
+  "service": "app",
+  "workspaceFolder": "/src",
   "postCreateCommand": "curl -fsSL https://pixi.sh/install.sh | bash && echo 'export PATH=\"$HOME/.pixi/bin:$PATH\"' >> ~/.bashrc"
-}
+}

.devcontainer/docker-compose.yml

@@ -1,30 +1,14 @@
+# Devcontainer overrides for the app service from ../docker-compose.yaml
+# Inherits db and minio services automatically
 services:
-  # Update this to the name of the service you want to work with in your docker-compose.yml file
   app:
-    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
-    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
-    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
-    # array). The sample below assumes your primary file is in the root of your project.
     container_name: datajoint-python-devcontainer
-    image: datajoint/datajoint-python-devcontainer:${PY_VER:-3.11}-${DISTRO:-bookworm}
     build:
-      context: .
+      context: ..
       dockerfile: .devcontainer/Dockerfile
       args:
         - PY_VER=${PY_VER:-3.11}
         - DISTRO=${DISTRO:-bookworm}
-
-    volumes:
-      # Update this to wherever you want VS Code to mount the folder of your project
-      - ..:/workspaces:cached
-
-    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
-    # cap_add:
-    #   - SYS_PTRACE
-    # security_opt:
-    #   - seccomp:unconfined
-
     user: root
-
-    # Overrides default command so things don't shut down after the process ends.
+    # Keep container running for devcontainer
     command: /bin/sh -c "while sleep 1000; do :; done"

.gitignore

@@ -187,3 +187,4 @@ dj_local_conf.json
 !.vscode/launch.json
 # pixi environments
 .pixi
+_content/

README.md

@@ -141,3 +141,73 @@ DataJoint (<https://datajoint.com>).
   - [Contribution Guidelines](https://docs.datajoint.com/about/contribute/)
 
   - [Developer Guide](https://docs.datajoint.com/core/datajoint-python/latest/develop/)
+
+## Developer Guide
+
+### Prerequisites
+
+- [Docker](https://docs.docker.com/get-docker/) for MySQL and MinIO services
+- Python 3.10+
+
+### Running Tests
+
+Tests are organized into `unit/` (no external services) and `integration/` (requires MySQL + MinIO):
+
+```bash
+# Install dependencies
+pip install -e ".[test]"
+
+# Run unit tests only (fast, no Docker needed)
+pytest tests/unit/
+
+# Start MySQL and MinIO for integration tests
+docker compose up -d db minio
+
+# Run all tests
+pytest tests/
+
+# Run specific test file
+pytest tests/integration/test_blob.py -v
+
+# Stop services when done
+docker compose down
+```
+
+### Alternative: Full Docker
+
+Run tests entirely in Docker (no local Python needed):
+
+```bash
+docker compose --profile test up djtest --build
+```
+
+### Alternative: Using pixi
+
+[pixi](https://pixi.sh) users can run tests with automatic service management:
+
+```bash
+pixi install        # First time setup
+pixi run test       # Starts services and runs tests
+pixi run services-down  # Stop services
+```
+
+### Pre-commit Hooks
+
+```bash
+pre-commit install          # Install hooks (first time)
+pre-commit run --all-files  # Run all checks
+```
+
+### Environment Variables
+
+Tests use these defaults (configured in `pyproject.toml`):
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DJ_HOST` | `localhost` | MySQL hostname |
+| `DJ_PORT` | `3306` | MySQL port |
+| `DJ_USER` | `root` | MySQL username |
+| `DJ_PASS` | `password` | MySQL password |
+| `S3_ENDPOINT` | `localhost:9000` | MinIO endpoint |
+
+For Docker-based testing (devcontainer, djtest), set `DJ_HOST=db` and `S3_ENDPOINT=minio:9000`.

docker-compose.yaml

@@ -1,15 +1,19 @@
 # Development environment with MySQL and MinIO services
-# To run tests: pytest --cov-report term-missing --cov=datajoint tests
+#
+# Quick start:
+#   docker compose up -d db minio   # Start services
+#   pytest tests/                    # Run tests (uses localhost defaults)
+#
+# Full Docker testing:
+#   docker compose --profile test up djtest --build
 services:
   db:
     image: datajoint/mysql:${MYSQL_VER:-8.0}
     environment:
       - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
     command: mysqld --default-authentication-plugin=mysql_native_password
-    # ports:
-    #   - "3306:3306"
-    # volumes:
-    #   - ./mysql/data:/var/lib/mysql
+    ports:
+      - "3306:3306"
     healthcheck:
       test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
       timeout: 30s
@@ -20,18 +24,15 @@ services:
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
-    # ports:
-    #   - "9000:9000"
-    # volumes:
-    #   - ./minio/config:/root/.minio
-    #   - ./minio/data:/data
+    ports:
+      - "9000:9000"
     command: server --address ":9000" /data
     healthcheck:
       test:
         - "CMD"
         - "curl"
         - "--fail"
-        - "http://minio:9000/minio/health/live"
+        - "http://localhost:9000/minio/health/live"
       timeout: 30s
       retries: 5
       interval: 15s

docs/src/compute/key-source.md

@@ -45,7 +45,7 @@ definition = """
 -> Recording
 ---
 sample_rate : float
-eeg_data : longblob
+eeg_data : <djblob>
 """
 key_source = Recording & 'recording_type = "EEG"'
 ```

docs/src/compute/make.md

@@ -152,7 +152,7 @@ class ImageAnalysis(dj.Computed):
     # Complex image analysis results
     -> Image
     ---
-    analysis_result : longblob
+    analysis_result : <djblob>
     processing_time : float
     """
 
@@ -188,7 +188,7 @@ class ImageAnalysis(dj.Computed):
     # Complex image analysis results
     -> Image
     ---
-    analysis_result : longblob
+    analysis_result : <djblob>
     processing_time : float
     """
 

docs/src/compute/populate.md

@@ -40,7 +40,7 @@ class FilteredImage(dj.Computed):
      # Filtered image
      -> Image
      ---
-     filtered_image : longblob
+     filtered_image : <djblob>
      """
 
      def make(self, key):
@@ -196,7 +196,7 @@ class ImageAnalysis(dj.Computed):
     # Complex image analysis results
     -> Image
     ---
-    analysis_result : longblob
+    analysis_result : <djblob>
     processing_time : float
     """
 
@@ -230,7 +230,7 @@ class ImageAnalysis(dj.Computed):
     # Complex image analysis results
     -> Image
     ---
-    analysis_result : longblob
+    analysis_result : <djblob>
     processing_time : float
     """
 

docs/src/design/integrity.md

@@ -142,7 +142,7 @@ definition = """
 -> EEGRecording
 channel_idx : int
 ---
-channel_data : longblob
+channel_data : <djblob>
 """
 ```
 ![doc_1-many](../images/doc_1-many.png){: style="align:center"}

docs/src/design/tables/attributes.md

@@ -48,9 +48,10 @@ fractional digits.
    Because of its well-defined precision, `decimal` values can be used in equality
    comparison and be included in primary keys.
 
--  `longblob`: arbitrary numeric array (e.g. matrix, image, structure), up to 4
+-  `longblob`: raw binary data, up to 4
 [GiB](http://en.wikipedia.org/wiki/Gibibyte) in size.
-   Numeric arrays are compatible between MATLAB and Python (NumPy).
+   Stores and returns raw bytes without serialization.
+   For serialized Python objects (arrays, dicts, etc.), use `<djblob>` instead.
    The `longblob` and other `blob` datatypes can be configured to store data
    [externally](../../sysadmin/external-store.md) by using the `blob@store` syntax.
 
@@ -71,6 +72,10 @@ info).
 These types abstract certain kinds of non-database data to facilitate use
 together with DataJoint.
 
+- `<djblob>`: DataJoint's native serialization format for Python objects. Supports
+NumPy arrays, dicts, lists, datetime objects, and nested structures. Compatible with
+MATLAB. See [custom types](customtype.md) for details.
+
 - `object`: managed [file and folder storage](object.md) with support for direct writes
 (Zarr, HDF5) and fsspec integration. Recommended for new pipelines.
 
@@ -80,6 +85,10 @@ sending/receiving an opaque data file to/from a DataJoint pipeline.
 - `filepath@store`: a [filepath](filepath.md) used to link non-DataJoint managed files
 into a DataJoint pipeline.
 
+- `<custom_type>`: a [custom attribute type](customtype.md) that defines bidirectional
+conversion between Python objects and database storage formats. Use this to store
+complex data types like graphs, domain-specific objects, or custom data structures.
+
 ## Numeric type aliases
 
 DataJoint provides convenient type aliases that map to standard MySQL numeric types.