ep

def print_data(data: np.ndarray):

print(data)@pipeline

def printing_pipeline():

# One can also pass data directly into the ExternalArtifact

# to create a new artifact on the fly

data = ExternalArtifact(value=np.array([0]))

print_data(data=data)

if __name__ == "__main__":

printing_pipeline()

Optionally, you can configure the ExternalArtifact to use a custom materializer for your data or disable artifact metadata and visualizations. Check out the SDK docs for all available options.

Using an ExternalArtifact for your step automatically disables caching for the step.

Consuming artifacts produced by other pipelines

It is also common to consume an artifact downstream after producing it in an upstream pipeline or step. As we have learned in the previous section, the Client can be used to fetch artifacts directly inside the pipeline code:

from uuid import UUID

import pandas as pd

from zenml import step, pipeline

from zenml.client import Client

@step

def trainer(dataset: pd.DataFrame):

...

@pipeline

def training_pipeline():

client = Client()

# Fetch by ID

dataset_artifact = client.get_artifact_version(

name_id_or_prefix=UUID("3a92ae32-a764-4420-98ba-07da8f742b76")

# Fetch by name alone - uses the latest version of this artifact

dataset_artifact = client.get_artifact_version(name_id_or_prefix="iris_dataset")

# Fetch by name and version

dataset_artifact = client.get_artifact_version(

name_id_or_prefix="iris_dataset", version="raw_2023"

# Pass into any step

trainer(dataset=dataset_artifact)

if __name__ == "__main__":

training_pipeline()

Calls of Client methods like get_artifact_version directly inside the pipeline code makes use of ZenML's late materialization behind the scenes.

If you would like to bypass materialization entirely and just download the data or files associated with a particular artifact version, you can use the .download_files method:

from zenml.client import Client

client = Client()

artifact = client.get_artifact_version(name_id_or_prefix="iris_dataset")