#!/usr/bin/env bash # Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh # ## reading ArrowStream file from python # import pyarrow as pa # stream = pa.ipc.open_stream("test.arrows") # x = stream.read_all() # print(x) ## writing ArrowStream file from python # import pyarrow as pa #data = [ # pa.array([1, 2, 3, 4, 5]), # pa.array(["onee", "twoo", "three", "four", "five"]).dictionary_encode(), # pa.array([1, 2, 3, 4, 5]).dictionary_encode(), # pa.array([True, False, True, True, True]) #] # batch = pa.record_batch(data, names=['id', 'lc_nullable', 'lc_int_nullable', 'bool_nullable']) # writer = pa.ipc.new_stream("test4.arrows", batch.schema) # writer.write_batch(batch) # writer.close() # cat data.arrow | gzip | base64 cat <