From 34c17b840d5bdb8060ef6309aee04f919616c9de Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 2 Apr 2026 09:31:14 +0200 Subject: [PATCH 1/5] reftable: introduce "reftable-system.h" header We're including a couple of standard headers like in a bunch of locations, which makes it hard for a project to plug in their own logic for making required functionality available. For us this is for example via "compat/posix.h", which already includes all of the system headers relevant to us. Introduce a new "reftable-system.h" header that allows projects to provide their own headers. This new header is supposed to contain all the project-specific bits to provide the POSIX-like environment, and some additional supporting code. With this change, we thus have the following split in our system-specific code: - "reftable/reftable-system.h" is the project-specific header that provides a POSIX-like environment. Every project is expected to provide their own implementation. - "reftable/system.h" contains the project-independent definition of the interfaces that a project needs to implement. This file should not be touched by a project. - "reftable/system.c" contains the project-specific implementation of the interfaces defined in "system.h". Again, every project is expected to provide their own implementation. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- reftable/reftable-basics.h | 2 +- reftable/reftable-block.h | 3 +-- reftable/reftable-blocksource.h | 2 +- reftable/reftable-error.h | 2 ++ reftable/reftable-fsck.h | 1 + reftable/reftable-iterator.h | 1 + reftable/reftable-merged.h | 1 + reftable/reftable-record.h | 2 +- reftable/reftable-stack.h | 1 + reftable/reftable-system.h | 15 +++++++++++++++ reftable/reftable-table.h | 1 + reftable/reftable-writer.h | 4 +--- reftable/system.h | 11 +++++++---- 13 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 reftable/reftable-system.h diff --git a/reftable/reftable-basics.h b/reftable/reftable-basics.h index 6d73f19c85..dc8622682d 100644 --- a/reftable/reftable-basics.h +++ b/reftable/reftable-basics.h @@ -9,7 +9,7 @@ #ifndef REFTABLE_BASICS_H #define REFTABLE_BASICS_H -#include +#include "reftable-system.h" /* A buffer that contains arbitrary byte slices. */ struct reftable_buf { diff --git a/reftable/reftable-block.h b/reftable/reftable-block.h index 0b05a8f7e3..94c79b5c58 100644 --- a/reftable/reftable-block.h +++ b/reftable/reftable-block.h @@ -9,8 +9,7 @@ #ifndef REFTABLE_BLOCK_H #define REFTABLE_BLOCK_H -#include - +#include "reftable-system.h" #include "reftable-basics.h" #include "reftable-blocksource.h" #include "reftable-iterator.h" diff --git a/reftable/reftable-blocksource.h b/reftable/reftable-blocksource.h index f5ba867bd6..40c1e94646 100644 --- a/reftable/reftable-blocksource.h +++ b/reftable/reftable-blocksource.h @@ -9,7 +9,7 @@ #ifndef REFTABLE_BLOCKSOURCE_H #define REFTABLE_BLOCKSOURCE_H -#include +#include "reftable-system.h" /* * Generic wrapper for a seekable readable file. diff --git a/reftable/reftable-error.h b/reftable/reftable-error.h index d100e0df92..0535e1478b 100644 --- a/reftable/reftable-error.h +++ b/reftable/reftable-error.h @@ -9,6 +9,8 @@ #ifndef REFTABLE_ERROR_H #define REFTABLE_ERROR_H +#include "reftable-system.h" + /* * Errors in reftable calls are signaled with negative integer return values. 0 * means success. diff --git a/reftable/reftable-fsck.h b/reftable/reftable-fsck.h index 007a392cf9..340fc7762e 100644 --- a/reftable/reftable-fsck.h +++ b/reftable/reftable-fsck.h @@ -1,6 +1,7 @@ #ifndef REFTABLE_FSCK_H #define REFTABLE_FSCK_H +#include "reftable-system.h" #include "reftable-stack.h" enum reftable_fsck_error { diff --git a/reftable/reftable-iterator.h b/reftable/reftable-iterator.h index af582028c2..a050cc153b 100644 --- a/reftable/reftable-iterator.h +++ b/reftable/reftable-iterator.h @@ -9,6 +9,7 @@ #ifndef REFTABLE_ITERATOR_H #define REFTABLE_ITERATOR_H +#include "reftable-system.h" #include "reftable-record.h" struct reftable_iterator_vtable; diff --git a/reftable/reftable-merged.h b/reftable/reftable-merged.h index e5af846b32..02a9966835 100644 --- a/reftable/reftable-merged.h +++ b/reftable/reftable-merged.h @@ -9,6 +9,7 @@ #ifndef REFTABLE_MERGED_H #define REFTABLE_MERGED_H +#include "reftable-system.h" #include "reftable-iterator.h" /* diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h index 385a74cc86..e18c538238 100644 --- a/reftable/reftable-record.h +++ b/reftable/reftable-record.h @@ -9,8 +9,8 @@ #ifndef REFTABLE_RECORD_H #define REFTABLE_RECORD_H +#include "reftable-system.h" #include "reftable-basics.h" -#include /* * Basic data types diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h index c2415cbc6e..5f7be573fa 100644 --- a/reftable/reftable-stack.h +++ b/reftable/reftable-stack.h @@ -9,6 +9,7 @@ #ifndef REFTABLE_STACK_H #define REFTABLE_STACK_H +#include "reftable-system.h" #include "reftable-writer.h" /* diff --git a/reftable/reftable-system.h b/reftable/reftable-system.h new file mode 100644 index 0000000000..4a18a6a790 --- /dev/null +++ b/reftable/reftable-system.h @@ -0,0 +1,15 @@ +#ifndef REFTABLE_SYSTEM_H +#define REFTABLE_SYSTEM_H + +/* + * This header defines the platform-specific bits required to compile the + * reftable library. It should provide an environment that bridges over the + * gaps between POSIX and your system, as well as the zlib interfaces. This + * header is expected to be changed by the individual project. + */ + +#define MINGW_DONT_HANDLE_IN_USE_ERROR +#include "compat/posix.h" +#include "compat/zlib-compat.h" + +#endif diff --git a/reftable/reftable-table.h b/reftable/reftable-table.h index 5f935d02e3..d7666b53a1 100644 --- a/reftable/reftable-table.h +++ b/reftable/reftable-table.h @@ -9,6 +9,7 @@ #ifndef REFTABLE_TABLE_H #define REFTABLE_TABLE_H +#include "reftable-system.h" #include "reftable-iterator.h" #include "reftable-block.h" #include "reftable-blocksource.h" diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h index 1e7003cd69..065dd93dc6 100644 --- a/reftable/reftable-writer.h +++ b/reftable/reftable-writer.h @@ -9,11 +9,9 @@ #ifndef REFTABLE_WRITER_H #define REFTABLE_WRITER_H +#include "reftable-system.h" #include "reftable-record.h" -#include -#include /* ssize_t */ - /* Writing single reftables */ /* reftable_write_options sets options for writing a single reftable. */ diff --git a/reftable/system.h b/reftable/system.h index c54ed4cad6..a7eb6acd4a 100644 --- a/reftable/system.h +++ b/reftable/system.h @@ -9,11 +9,14 @@ #ifndef SYSTEM_H #define SYSTEM_H -/* This header glues the reftable library to the rest of Git */ +/* + * This header defines the platform-agnostic interface that is to be + * implemented by the project to make it work on their respective supported + * systems, and to integrate it into the project itself. This header is not + * expected to be changed by the individual project. + */ -#define MINGW_DONT_HANDLE_IN_USE_ERROR -#include "compat/posix.h" -#include "compat/zlib-compat.h" +#include "reftable-system.h" /* * Return a random 32 bit integer. This function is expected to return From b45ea595e6f6b03a749abc2c8e508504429a4cf3 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 2 Apr 2026 09:31:15 +0200 Subject: [PATCH 2/5] reftable/stack: provide fsync(3p) via system header Users of the reftable library are expected to provide their own function callback in cases they want to sync(3p) data to disk via the reftable write options. But if no such function was provided we end up calling fsync(3p) directly, which may not even be available on some systems. While dropping the explicit call to fsync(3p) would work, it would lead to an unsafe default behaviour where a project may have forgotten to set up the callback function, and that could lead to potential data loss. So this is not a great solution. Instead, drop the callback function and make it mandatory for the project to define fsync(3p). In the case of Git, we can then easily inject our custom implementation via the "reftable-system.h" header so that we continue to use `fsync_component()`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- refs/reftable-backend.c | 6 ------ reftable/reftable-system.h | 3 +++ reftable/reftable-writer.h | 6 ------ reftable/stack.c | 13 +++---------- reftable/system.c | 6 ++++++ 5 files changed, 12 insertions(+), 22 deletions(-) diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index b124404663..daea30a5b4 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -366,11 +366,6 @@ static int reftable_be_config(const char *var, const char *value, return 0; } -static int reftable_be_fsync(int fd) -{ - return fsync_component(FSYNC_COMPONENT_REFERENCE, fd); -} - static struct ref_store *reftable_be_init(struct repository *repo, const char *payload, const char *gitdir, @@ -408,7 +403,6 @@ static struct ref_store *reftable_be_init(struct repository *repo, refs->write_options.disable_auto_compact = !git_env_bool("GIT_TEST_REFTABLE_AUTOCOMPACTION", 1); refs->write_options.lock_timeout_ms = 100; - refs->write_options.fsync = reftable_be_fsync; repo_config(the_repository, reftable_be_config, &refs->write_options); diff --git a/reftable/reftable-system.h b/reftable/reftable-system.h index 4a18a6a790..76f3e33e90 100644 --- a/reftable/reftable-system.h +++ b/reftable/reftable-system.h @@ -12,4 +12,7 @@ #include "compat/posix.h" #include "compat/zlib-compat.h" +int reftable_fsync(int fd); +#define fsync(fd) reftable_fsync(fd) + #endif diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h index 065dd93dc6..a66db415c8 100644 --- a/reftable/reftable-writer.h +++ b/reftable/reftable-writer.h @@ -61,12 +61,6 @@ struct reftable_write_options { */ long lock_timeout_ms; - /* - * Optional callback used to fsync files to disk. Falls back to using - * fsync(3P) when unset. - */ - int (*fsync)(int fd); - /* * Callback function to execute whenever the stack is being reloaded. * This can be used e.g. to discard cached information that relies on diff --git a/reftable/stack.c b/reftable/stack.c index 1c9f21dfe1..fa87b46c37 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -29,13 +29,6 @@ static int stack_filename(struct reftable_buf *dest, struct reftable_stack *st, return 0; } -static int stack_fsync(const struct reftable_write_options *opts, int fd) -{ - if (opts->fsync) - return opts->fsync(fd); - return fsync(fd); -} - static ssize_t reftable_write_data(int fd, const void *data, size_t size) { size_t total_written = 0; @@ -69,7 +62,7 @@ static ssize_t fd_writer_write(void *arg, const void *data, size_t sz) static int fd_writer_flush(void *arg) { struct fd_writer *writer = arg; - return stack_fsync(writer->opts, writer->fd); + return fsync(writer->fd); } static int fd_read_lines(int fd, char ***namesp) @@ -812,7 +805,7 @@ int reftable_addition_commit(struct reftable_addition *add) goto done; } - err = stack_fsync(&add->stack->opts, add->tables_list_lock.fd); + err = fsync(add->tables_list_lock.fd); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1480,7 +1473,7 @@ static int stack_compact_range(struct reftable_stack *st, goto done; } - err = stack_fsync(&st->opts, tables_list_lock.fd); + err = fsync(tables_list_lock.fd); if (err < 0) { err = REFTABLE_IO_ERROR; unlink(new_table_path.buf); diff --git a/reftable/system.c b/reftable/system.c index 725a25844e..4d7e366b55 100644 --- a/reftable/system.c +++ b/reftable/system.c @@ -5,6 +5,7 @@ #include "reftable-error.h" #include "../lockfile.h" #include "../tempfile.h" +#include "../write-or-die.h" uint32_t reftable_rand(void) { @@ -131,3 +132,8 @@ int flock_commit(struct reftable_flock *l) return 0; } + +int reftable_fsync(int fd) +{ + return fsync_component(FSYNC_COMPONENT_REFERENCE, fd); +} From aa8938573050e6ab44a46e3b9f26c0e442f835aa Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 2 Apr 2026 09:31:16 +0200 Subject: [PATCH 3/5] reftable/fsck: use REFTABLE_UNUSED instead of UNUSED While we have the reftable-specific `REFTABLE_UNUSED` header, we accidentally introduced a new usage of the Git-specific `UNUSED` header into the reftable library in 9051638519 (reftable: add code to facilitate consistency checks, 2025-10-07). Convert the site to use `REFTABLE_UNUSED`. Ideally, we'd move the definition of `UNUSED` into "git-compat-util.h" so that it becomes in accessible to the reftable library. But this is unfortunately not easily possible as "compat/mingw-posix.h" requires this macro, and this header is included by "compat/posix.h". Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- reftable/fsck.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reftable/fsck.c b/reftable/fsck.c index 26b9115b14..8e73fc83f2 100644 --- a/reftable/fsck.c +++ b/reftable/fsck.c @@ -63,7 +63,7 @@ static int table_check_name(struct reftable_table *table, static int table_checks(struct reftable_table *table, reftable_fsck_report_fn report_fn, - reftable_fsck_verbose_fn verbose_fn UNUSED, + reftable_fsck_verbose_fn verbose_fn REFTABLE_UNUSED, void *cb_data) { table_check_fn table_check_fns[] = { From cb0882de1979522b2fc3dc4c3064b0ad21d50b06 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 2 Apr 2026 09:31:17 +0200 Subject: [PATCH 4/5] reftable/system: add abstraction to retrieve time in milliseconds We directly call gettimeofday(3p), which may not be available on some platforms. Provide the infrastructure to let projects easily use their own implementations of this function. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- reftable/stack.c | 27 ++++----------------------- reftable/system.c | 6 ++++++ reftable/system.h | 3 +++ 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/reftable/stack.c b/reftable/stack.c index fa87b46c37..1fba96ddb3 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -365,45 +365,26 @@ done: return err; } -/* return negative if a before b. */ -static int tv_cmp(struct timeval *a, struct timeval *b) -{ - time_t diff = a->tv_sec - b->tv_sec; - int udiff = a->tv_usec - b->tv_usec; - - if (diff != 0) - return diff; - - return udiff; -} - static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, int reuse_open) { char **names = NULL, **names_after = NULL; - struct timeval deadline; + uint64_t deadline; int64_t delay = 0; int tries = 0, err; int fd = -1; - err = gettimeofday(&deadline, NULL); - if (err < 0) - goto out; - deadline.tv_sec += 3; + deadline = reftable_time_ms() + 3000; while (1) { - struct timeval now; - - err = gettimeofday(&now, NULL); - if (err < 0) - goto out; + uint64_t now = reftable_time_ms(); /* * Only look at deadlines after the first few times. This * simplifies debugging in GDB. */ tries++; - if (tries > 3 && tv_cmp(&now, &deadline) >= 0) + if (tries > 3 && now >= deadline) goto out; fd = open(st->list_file, O_RDONLY); diff --git a/reftable/system.c b/reftable/system.c index 4d7e366b55..cd76e56be8 100644 --- a/reftable/system.c +++ b/reftable/system.c @@ -4,6 +4,7 @@ #include "basics.h" #include "reftable-error.h" #include "../lockfile.h" +#include "../trace.h" #include "../tempfile.h" #include "../write-or-die.h" @@ -137,3 +138,8 @@ int reftable_fsync(int fd) { return fsync_component(FSYNC_COMPONENT_REFERENCE, fd); } + +uint64_t reftable_time_ms(void) +{ + return getnanotime() / 1000000; +} diff --git a/reftable/system.h b/reftable/system.h index a7eb6acd4a..071bfa3d58 100644 --- a/reftable/system.h +++ b/reftable/system.h @@ -111,4 +111,7 @@ int flock_release(struct reftable_flock *l); */ int flock_commit(struct reftable_flock *l); +/* Report the time in milliseconds. */ +uint64_t reftable_time_ms(void); + #endif From 87e4eee3f94ec261a92a76d06261b227b00de461 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 2 Apr 2026 09:31:18 +0200 Subject: [PATCH 5/5] reftable/system: add abstraction to mmap files In our codebase we have a couple of wrappers around mmap(3p) that allow us to reimplement the syscall on platforms that don't have it natively, like for example Windows. Other projects that embed the reftable library may have a different infra though to hook up mmap wrappers, but these are currently hard to integrate. Provide the infrastructure to let projects easily define the mmap interface with a custom struct and custom functions. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- reftable/blocksource.c | 19 +++++++------------ reftable/system.c | 20 ++++++++++++++++++++ reftable/system.h | 18 ++++++++++++++++++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/reftable/blocksource.c b/reftable/blocksource.c index 573c81287f..7f7441f751 100644 --- a/reftable/blocksource.c +++ b/reftable/blocksource.c @@ -93,13 +93,12 @@ void block_source_from_buf(struct reftable_block_source *bs, } struct file_block_source { - uint64_t size; - unsigned char *data; + struct reftable_mmap mmap; }; static uint64_t file_size(void *b) { - return ((struct file_block_source *)b)->size; + return ((struct file_block_source *)b)->mmap.size; } static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest REFTABLE_UNUSED) @@ -109,7 +108,7 @@ static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_dat static void file_close(void *v) { struct file_block_source *b = v; - munmap(b->data, b->size); + reftable_munmap(&b->mmap); reftable_free(b); } @@ -117,8 +116,8 @@ static ssize_t file_read_data(void *v, struct reftable_block_data *dest, uint64_ uint32_t size) { struct file_block_source *b = v; - assert(off + size <= b->size); - dest->data = b->data + off; + assert(off + size <= b->mmap.size); + dest->data = (unsigned char *) b->mmap.data + off; dest->len = size; return size; } @@ -156,13 +155,9 @@ int reftable_block_source_from_file(struct reftable_block_source *bs, goto out; } - p->size = st.st_size; - p->data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - if (p->data == MAP_FAILED) { - err = REFTABLE_IO_ERROR; - p->data = NULL; + err = reftable_mmap(&p->mmap, fd, st.st_size); + if (err < 0) goto out; - } assert(!bs->ops); bs->ops = &file_vtable; diff --git a/reftable/system.c b/reftable/system.c index cd76e56be8..9063641f30 100644 --- a/reftable/system.c +++ b/reftable/system.c @@ -143,3 +143,23 @@ uint64_t reftable_time_ms(void) { return getnanotime() / 1000000; } + +int reftable_mmap(struct reftable_mmap *out, int fd, size_t len) +{ + void *data = xmmap_gently(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (data == MAP_FAILED) + return REFTABLE_IO_ERROR; + + out->data = data; + out->size = len; + + return 0; +} + +int reftable_munmap(struct reftable_mmap *mmap) +{ + if (munmap(mmap->data, mmap->size) < 0) + return REFTABLE_IO_ERROR; + memset(mmap, 0, sizeof(*mmap)); + return 0; +} diff --git a/reftable/system.h b/reftable/system.h index 071bfa3d58..c0e2cbe0ff 100644 --- a/reftable/system.h +++ b/reftable/system.h @@ -114,4 +114,22 @@ int flock_commit(struct reftable_flock *l); /* Report the time in milliseconds. */ uint64_t reftable_time_ms(void); +struct reftable_mmap { + void *data; + size_t size; + void *priv; +}; + +/* + * Map the file into memory. Returns 0 on success, a reftable error code on + * error. + */ +int reftable_mmap(struct reftable_mmap *out, int fd, size_t len); + +/* + * Unmap the file from memory. Returns 0 on success, a reftable error code on + * error. + */ +int reftable_munmap(struct reftable_mmap *mmap); + #endif