commit 4c72c067bfecd0daa9c622e2a08a761bc675b5c2
parent 86425c7d0d5337c4187b2e7e5de8c813a9e31cbd
Author: Drew DeVault <sir@cmpwn.com>
Date: Wed, 24 Mar 2021 12:35:39 -0400
hare::module: implement cache manifest loading
Diffstat:
4 files changed, 208 insertions(+), 8 deletions(-)
diff --git a/cmd/hare/schedule.ha b/cmd/hare/schedule.ha
@@ -155,7 +155,6 @@ fn sched_hare_object(
};
let output = if (len(namespace) != 0) {
- // TODO: consult/update cache manifest
let version = hex::encode(ver.hash);
let ns = unparse::ident_s(namespace);
let env = module::ident_underscore(namespace);
@@ -166,6 +165,15 @@ fn sched_hare_object(
fmt::asprintf("HARE_VERSION_{}", env), version,
));
+ let manifest = match (module::manifest_load(
+ plan.context, namespace)) {
+ err: module::error => fmt::fatal(
+ "Error reading cache entry for {}: {}",
+ ns, module::errstr(err)),
+ m: module::manifest => m,
+ };
+ defer module::manifest_finish(&manifest);
+
let name = fmt::asprintf("{}.{}", version,
if (mixed) "a" else "o");
defer free(name);
@@ -188,6 +196,7 @@ fn sched_hare_object(
append(harec.cmd, path);
};
};
+
append(plan.scheduled, harec);
let s = mkfile(plan, "s");
diff --git a/hare/module/manifest.ha b/hare/module/manifest.ha
@@ -1,10 +1,15 @@
+use bufio;
+use bytes;
use encoding::hex;
+use encoding::utf8;
use fmt;
use fs;
use hare::ast;
use hare::unparse;
use io;
use path;
+use strconv;
+use strings;
use time;
// The manifest file format is a series of line-oriented records. Lines starting
@@ -16,6 +21,173 @@ use time;
// - "module" is a version of a module, and includes the module hash and the set
// of input hashes which produce it.
+def VERSION: int = 1;
+
+fn getinput(m: *manifest, hash: []u8) nullable *input = {
+ for (let i = 0z; i < len(m.inputs); i += 1) {
+ if (bytes::equal(m.inputs[i].hash, hash)) {
+ return &m.inputs[i];
+ };
+ };
+ return null;
+};
+
+// Loads the module manifest from the build cache for the given ident. The
+// return value borrows the ident parameter. If the module is not found, an
+// empty manifest is returned.
+export fn manifest_load(ctx: *context, ident: ast::ident) (manifest | error) = {
+ let manifest = manifest {
+ ident = ident,
+ inputs = [],
+ versions = [],
+ };
+ let ipath = ident_path(manifest.ident);
+ defer free(ipath);
+ let cachedir = path::join(ctx.cache, ipath);
+ defer free(cachedir);
+
+ let mpath = path::join(cachedir, "manifest");
+ defer free(mpath);
+
+ // TODO: We can probably eliminate these locks by using atomic writes
+ // instead
+ let l = lock(ctx.fs, cachedir)?;
+ defer unlock(ctx.fs, cachedir, l);
+
+ let file = match (fs::open(ctx.fs, mpath, fs::flags::RDONLY)) {
+ fs::noentry => return manifest,
+ err: fs::error => return err,
+ file: *io::stream => file,
+ };
+ defer io::close(file);
+
+ let inputs: []input = [], versions: []version = [];
+
+ let buf: [4096]u8 = [0...];
+ let file = bufio::buffered(file, buf, []);
+ for (true) {
+ let line = match (bufio::scanline(file)?) {
+ io::EOF => break,
+ line: []u8 => match (strings::try_from_utf8(line)) {
+ // Treat an invalid manifest as empty
+ utf8::invalid => return manifest,
+ s: str => s,
+ },
+ };
+ defer free(line);
+
+ if (strings::has_prefix(line, "#")) {
+ continue;
+ };
+
+ let tok = strings::tokenize(line, " ");
+ let kind = match (strings::next_token(&tok)) {
+ void => continue,
+ s: str => s,
+ };
+
+ if (kind == "version") {
+ let ver = match (strings::next_token(&tok)) {
+ void => return manifest,
+ s: str => s,
+ };
+ match (strconv::stoi(ver)) {
+ v: int => if (v != VERSION) {
+ return manifest;
+ },
+ * => return manifest,
+ };
+ } else if (kind == "input") {
+ let hash = match (strings::next_token(&tok)) {
+ void => return manifest, s: str => s,
+ }, path = match (strings::next_token(&tok)) {
+ void => return manifest, s: str => s,
+ }, inode = match (strings::next_token(&tok)) {
+ void => return manifest, s: str => s,
+ }, mtime = match (strings::next_token(&tok)) {
+ void => return manifest, s: str => s,
+ };
+
+ let hash = match (hex::decode(hash)) {
+ * => return manifest,
+ b: []u8 => b,
+ };
+ let inode = match (strconv::stoz(inode)) {
+ * => return manifest,
+ z: size => z,
+ };
+ let mtime = match (strconv::stoi64(mtime)) {
+ * => return manifest,
+ i: i64 => time::from_unix(i),
+ };
+
+ let parsed = parse_name(path);
+ let ftype = match (type_for_ext(path)) {
+ void => return manifest,
+ ft: filetype => ft,
+ };
+
+ append(inputs, input {
+ hash = hash,
+ path = strings::dup(path),
+ ft = ftype,
+ stat = fs::filestat {
+ mask = fs::stat_mask::MTIME | fs::stat_mask::INODE,
+ mtime = mtime,
+ inode = inode,
+ },
+ basename = strings::dup(parsed.0),
+ tags = parsed.2,
+ });
+ } else if (kind == "module") {
+ let modhash = match (strings::next_token(&tok)) {
+ void => return manifest, s: str => s,
+ };
+ let modhash = match (hex::decode(modhash)) {
+ * => return manifest,
+ b: []u8 => b,
+ };
+
+ let inputs: []input = [];
+ for (true) {
+ let hash = match (strings::next_token(&tok)) {
+ void => break,
+ s: str => s,
+ };
+ let hash = match (hex::decode(hash)) {
+ * => return manifest,
+ b: []u8 => b,
+ };
+ defer free(hash);
+
+ let input = match (getinput(&manifest, hash)) {
+ null => return manifest,
+ i: *input => i,
+ };
+
+ append(inputs, *input);
+ };
+
+ append(versions, version {
+ hash = modhash,
+ inputs = inputs,
+ });
+ } else {
+ return manifest;
+ };
+
+ // Check for extra tokens
+ match (strings::next_token(&tok)) {
+ void => void,
+ s: str => return manifest,
+ };
+ };
+
+ manifest.inputs = inputs;
+ manifest.versions = versions;
+ return manifest;
+};
+
// Writes a module manifest to the build cache.
export fn manifest_write(ctx: *context, manifest: *manifest) (void | error) = {
let ipath = ident_path(manifest.ident);
@@ -37,7 +209,7 @@ export fn manifest_write(ctx: *context, manifest: *manifest) (void | error) = {
fmt::fprintfln(fd, "# {}", ident)?;
fmt::fprintln(fd, "# This file is an internal Hare implementation detail.")?;
fmt::fprintln(fd, "# The format is not stable.")?;
- fmt::fprintln(fd, "version 1")?;
+ fmt::fprintfln(fd, "version {}", VERSION)?;
for (let i = 0z; i < len(manifest.inputs); i += 1) {
const input = manifest.inputs[i];
let hash = hex::encode(input.hash);
@@ -99,3 +271,21 @@ fn unlock(fs: *fs::fs, cachedir: str, s: *io::stream) void = {
err: fs::error => abort("Error removing module lock"),
};
};
+
+fn input_finish(in: *input) void = {
+ free(in.hash);
+ free(in.path);
+ free(in.basename);
+ tags_free(in.tags);
+};
+
+// Frees resources associated with this manifest.
+export fn manifest_finish(m: *manifest) void = {
+ for (let i = 0z; i < len(m.inputs); i += 1) {
+ input_finish(&m.inputs[i]);
+ };
+
+ for (let i = 0z; i < len(m.versions); i += 1) {
+ free(m.versions[i].inputs);
+ };
+};
diff --git a/scripts/gen-stdlib b/scripts/gen-stdlib
@@ -188,7 +188,7 @@ encoding_hex() {
printf '# encoding::hex\n'
gen_srcs encoding::hex \
hex.ha
- gen_ssa encoding::hex io strconv strio strings
+ gen_ssa encoding::hex io strconv strio strings bytes
}
encoding_utf8() {
@@ -294,7 +294,8 @@ hare_module() {
manifest.ha
gen_ssa hare::module \
hare::ast hare::lex hare::parse hare::unparse strio fs io strings hash \
- crypto::sha256 dirs bytes encoding::utf8 ascii fmt time slice
+ crypto::sha256 dirs bytes encoding::utf8 ascii fmt time slice bufio \
+ strconv
}
gensrcs_hare_parse() {
diff --git a/stdlib.mk b/stdlib.mk
@@ -276,7 +276,7 @@ $(HARECACHE)/dirs/dirs.ssa: $(stdlib_dirs_srcs) $(stdlib_rt) $(stdlib_fs) $(stdl
stdlib_encoding_hex_srcs= \
$(STDLIB)/encoding/hex/hex.ha
-$(HARECACHE)/encoding/hex/encoding_hex.ssa: $(stdlib_encoding_hex_srcs) $(stdlib_rt) $(stdlib_io) $(stdlib_strconv) $(stdlib_strio) $(stdlib_strings)
+$(HARECACHE)/encoding/hex/encoding_hex.ssa: $(stdlib_encoding_hex_srcs) $(stdlib_rt) $(stdlib_io) $(stdlib_strconv) $(stdlib_strio) $(stdlib_strings) $(stdlib_bytes)
@printf 'HAREC \t$@\n'
@mkdir -p $(HARECACHE)/encoding/hex
@HARECACHE=$(HARECACHE) $(HAREC) $(HAREFLAGS) -o $@ -Nencoding::hex \
@@ -383,7 +383,7 @@ stdlib_hare_module_srcs= \
$(STDLIB)/hare/module/scan.ha \
$(STDLIB)/hare/module/manifest.ha
-$(HARECACHE)/hare/module/hare_module.ssa: $(stdlib_hare_module_srcs) $(stdlib_rt) $(stdlib_hare_ast) $(stdlib_hare_lex) $(stdlib_hare_parse) $(stdlib_hare_unparse) $(stdlib_strio) $(stdlib_fs) $(stdlib_io) $(stdlib_strings) $(stdlib_hash) $(stdlib_crypto_sha256) $(stdlib_dirs) $(stdlib_bytes) $(stdlib_encoding_utf8) $(stdlib_ascii) $(stdlib_fmt) $(stdlib_time) $(stdlib_slice)
+$(HARECACHE)/hare/module/hare_module.ssa: $(stdlib_hare_module_srcs) $(stdlib_rt) $(stdlib_hare_ast) $(stdlib_hare_lex) $(stdlib_hare_parse) $(stdlib_hare_unparse) $(stdlib_strio) $(stdlib_fs) $(stdlib_io) $(stdlib_strings) $(stdlib_hash) $(stdlib_crypto_sha256) $(stdlib_dirs) $(stdlib_bytes) $(stdlib_encoding_utf8) $(stdlib_ascii) $(stdlib_fmt) $(stdlib_time) $(stdlib_slice) $(stdlib_bufio) $(stdlib_strconv)
@printf 'HAREC \t$@\n'
@mkdir -p $(HARECACHE)/hare/module
@HARECACHE=$(HARECACHE) $(HAREC) $(HAREFLAGS) -o $@ -Nhare::module \
@@ -927,7 +927,7 @@ $(TESTCACHE)/dirs/dirs.ssa: $(testlib_dirs_srcs) $(testlib_rt) $(testlib_fs) $(t
testlib_encoding_hex_srcs= \
$(STDLIB)/encoding/hex/hex.ha
-$(TESTCACHE)/encoding/hex/encoding_hex.ssa: $(testlib_encoding_hex_srcs) $(testlib_rt) $(testlib_io) $(testlib_strconv) $(testlib_strio) $(testlib_strings)
+$(TESTCACHE)/encoding/hex/encoding_hex.ssa: $(testlib_encoding_hex_srcs) $(testlib_rt) $(testlib_io) $(testlib_strconv) $(testlib_strio) $(testlib_strings) $(testlib_bytes)
@printf 'HAREC \t$@\n'
@mkdir -p $(TESTCACHE)/encoding/hex
@HARECACHE=$(TESTCACHE) $(HAREC) $(TESTHAREFLAGS) -o $@ -Nencoding::hex \
@@ -1035,7 +1035,7 @@ testlib_hare_module_srcs= \
$(STDLIB)/hare/module/scan.ha \
$(STDLIB)/hare/module/manifest.ha
-$(TESTCACHE)/hare/module/hare_module.ssa: $(testlib_hare_module_srcs) $(testlib_rt) $(testlib_hare_ast) $(testlib_hare_lex) $(testlib_hare_parse) $(testlib_hare_unparse) $(testlib_strio) $(testlib_fs) $(testlib_io) $(testlib_strings) $(testlib_hash) $(testlib_crypto_sha256) $(testlib_dirs) $(testlib_bytes) $(testlib_encoding_utf8) $(testlib_ascii) $(testlib_fmt) $(testlib_time) $(testlib_slice)
+$(TESTCACHE)/hare/module/hare_module.ssa: $(testlib_hare_module_srcs) $(testlib_rt) $(testlib_hare_ast) $(testlib_hare_lex) $(testlib_hare_parse) $(testlib_hare_unparse) $(testlib_strio) $(testlib_fs) $(testlib_io) $(testlib_strings) $(testlib_hash) $(testlib_crypto_sha256) $(testlib_dirs) $(testlib_bytes) $(testlib_encoding_utf8) $(testlib_ascii) $(testlib_fmt) $(testlib_time) $(testlib_slice) $(testlib_bufio) $(testlib_strconv)
@printf 'HAREC \t$@\n'
@mkdir -p $(TESTCACHE)/hare/module
@HARECACHE=$(TESTCACHE) $(HAREC) $(TESTHAREFLAGS) -o $@ -Nhare::module \
