git/odb/streaming.c

/*
 * Copyright (c) 2011, Google Inc.
 */

#include "git-compat-util.h"
#include "convert.h"
#include "environment.h"
#include "repository.h"
#include "odb.h"
#include "odb/source.h"
#include "odb/streaming.h"
#include "replace-object.h"

#define FILTER_BUFFER (1024*16)

/*****************************************************************
 *
 * Filtered stream
 *
 *****************************************************************/

struct odb_filtered_read_stream {
	struct odb_read_stream base;
	struct odb_read_stream *upstream;
	struct stream_filter *filter;
	char ibuf[FILTER_BUFFER];
	char obuf[FILTER_BUFFER];
	int i_end, i_ptr;
	int o_end, o_ptr;
	int input_finished;
};

static int close_istream_filtered(struct odb_read_stream *_fs)
{
	struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs;
	free_stream_filter(fs->filter);
	return odb_read_stream_close(fs->upstream);
}

static ssize_t read_istream_filtered(struct odb_read_stream *_fs, char *buf,
				     size_t sz)
{
	struct odb_filtered_read_stream *fs = (struct odb_filtered_read_stream *)_fs;
	size_t filled = 0;

	while (sz) {
		/* do we already have filtered output? */
		if (fs->o_ptr < fs->o_end) {
			size_t to_move = fs->o_end - fs->o_ptr;
			if (sz < to_move)
				to_move = sz;
			memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move);
			fs->o_ptr += to_move;
			sz -= to_move;
			filled += to_move;
			continue;
		}
		fs->o_end = fs->o_ptr = 0;

		/* do we have anything to feed the filter with? */
		if (fs->i_ptr < fs->i_end) {
			size_t to_feed = fs->i_end - fs->i_ptr;
			size_t to_receive = FILTER_BUFFER;
			if (stream_filter(fs->filter,
					  fs->ibuf + fs->i_ptr, &to_feed,
					  fs->obuf, &to_receive))
				return -1;
			fs->i_ptr = fs->i_end - to_feed;
			fs->o_end = FILTER_BUFFER - to_receive;
			continue;
		}

		/* tell the filter to drain upon no more input */
		if (fs->input_finished) {
			size_t to_receive = FILTER_BUFFER;
			if (stream_filter(fs->filter,
					  NULL, NULL,
					  fs->obuf, &to_receive))
				return -1;
			fs->o_end = FILTER_BUFFER - to_receive;
			if (!fs->o_end)
				break;
			continue;
		}
		fs->i_end = fs->i_ptr = 0;

		/* refill the input from the upstream */
		if (!fs->input_finished) {
			fs->i_end = odb_read_stream_read(fs->upstream, fs->ibuf, FILTER_BUFFER);
			if (fs->i_end < 0)
				return -1;
			if (fs->i_end)
				continue;
		}
		fs->input_finished = 1;
	}
	return filled;
}

static struct odb_read_stream *attach_stream_filter(struct odb_read_stream *st,
						    struct stream_filter *filter)
{
	struct odb_filtered_read_stream *fs;

	CALLOC_ARRAY(fs, 1);
	fs->base.close = close_istream_filtered;
	fs->base.read = read_istream_filtered;
	fs->upstream = st;
	fs->filter = filter;
	fs->base.size = -1; /* unknown */
	fs->base.type = st->type;

	return &fs->base;
}

/*****************************************************************
 *
 * In-core stream
 *
 *****************************************************************/

struct odb_incore_read_stream {
	struct odb_read_stream base;
	char *buf; /* from odb_read_object_info_extended() */
	unsigned long read_ptr;
};

static int close_istream_incore(struct odb_read_stream *_st)
{
	struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st;
	free(st->buf);
	return 0;
}

static ssize_t read_istream_incore(struct odb_read_stream *_st, char *buf, size_t sz)
{
	struct odb_incore_read_stream *st = (struct odb_incore_read_stream *)_st;
	size_t read_size = sz;
	size_t remainder = st->base.size - st->read_ptr;

	if (remainder <= read_size)
		read_size = remainder;
	if (read_size) {
		memcpy(buf, st->buf + st->read_ptr, read_size);
		st->read_ptr += read_size;
	}
	return read_size;
}

static int open_istream_incore(struct odb_read_stream **out,
			       struct object_database *odb,
			       const struct object_id *oid)
{
	struct object_info oi = OBJECT_INFO_INIT;
	struct odb_incore_read_stream stream = {
		.base.close = close_istream_incore,
		.base.read = read_istream_incore,
	};
	struct odb_incore_read_stream *st;
	unsigned long size_ul;
	int ret;

	oi.typep = &stream.base.type;
	/*
	 * object_info.sizep is unsigned long* (32-bit on Windows), but
	 * stream.base.size is size_t (64-bit). We use a temporary variable
	 * because the types are incompatible. Note: this path still truncates
	 * for >4GB objects, but large objects should use pack streaming
	 * (packfile_store_read_object_stream) which handles size_t properly.
	 * This incore fallback is only used for small objects or when pack
	 * streaming is unavailable.
	 */
	oi.sizep = &size_ul;
	oi.contentp = (void **)&stream.buf;
	ret = odb_read_object_info_extended(odb, oid, &oi,
					    OBJECT_INFO_DIE_IF_CORRUPT);
	if (ret)
		return ret;
	stream.base.size = size_ul;

	CALLOC_ARRAY(st, 1);
	*st = stream;
	*out = &st->base;

	return 0;
}

/*****************************************************************************
 * static helpers variables and functions for users of streaming interface
 *****************************************************************************/

static int istream_source(struct odb_read_stream **out,
			  struct object_database *odb,
			  const struct object_id *oid)
{
	struct odb_source *source;

	odb_prepare_alternates(odb);
	for (source = odb->sources; source; source = source->next)
		if (!odb_source_read_object_stream(out, source, oid))
			return 0;

	return open_istream_incore(out, odb, oid);
}

/****************************************************************
 * Users of streaming interface
 ****************************************************************/

int odb_read_stream_close(struct odb_read_stream *st)
{
	int r = st->close(st);
	free(st);
	return r;
}

ssize_t odb_read_stream_read(struct odb_read_stream *st, void *buf, size_t sz)
{
	return st->read(st, buf, sz);
}

struct odb_read_stream *odb_read_stream_open(struct object_database *odb,
					     const struct object_id *oid,
					     struct stream_filter *filter)
{
	struct odb_read_stream *st;
	const struct object_id *real = lookup_replace_object(odb->repo, oid);
	int ret = istream_source(&st, odb, real);

	if (ret)
		return NULL;

	if (filter) {
		/* Add "&& !is_null_stream_filter(filter)" for performance */
		struct odb_read_stream *nst = attach_stream_filter(st, filter);
		if (!nst) {
			odb_read_stream_close(st);
			return NULL;
		}
		st = nst;
	}

	return st;
}

int odb_stream_blob_to_fd(struct object_database *odb,
			  int fd,
			  const struct object_id *oid,
			  struct stream_filter *filter,
			  int can_seek)
{
	struct odb_read_stream *st;
	ssize_t kept = 0;
	int result = -1;

	st = odb_read_stream_open(odb, oid, filter);
	if (!st) {
		if (filter)
			free_stream_filter(filter);
		return result;
	}
	if (st->type != OBJ_BLOB)
		goto close_and_exit;
	for (;;) {
		char buf[1024 * 16];
		ssize_t wrote, holeto;
		ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf));

		if (readlen < 0)
			goto close_and_exit;
		if (!readlen)
			break;
		if (can_seek && sizeof(buf) == readlen) {
			for (holeto = 0; holeto < readlen; holeto++)
				if (buf[holeto])
					break;
			if (readlen == holeto) {
				kept += holeto;
				continue;
			}
		}

		if (kept && lseek(fd, kept, SEEK_CUR) == (off_t) -1)
			goto close_and_exit;
		else
			kept = 0;
		wrote = write_in_full(fd, buf, readlen);

		if (wrote < 0)
			goto close_and_exit;
	}
	if (kept && (lseek(fd, kept - 1, SEEK_CUR) == (off_t) -1 ||
		     xwrite(fd, "", 1) != 1))
		goto close_and_exit;
	result = 0;

 close_and_exit:
	odb_read_stream_close(st);
	return result;
}