harec

[hare] Hare compiler, written in C11 for POSIX OSs
Log | Files | Refs | README | LICENSE

gen.c (117619B)


      1 #include <assert.h>
      2 #include <stdlib.h>
      3 #include <string.h>
      4 #include "check.h"
      5 #include "expr.h"
      6 #include "gen.h"
      7 #include "scope.h"
      8 #include "typedef.h"
      9 #include "types.h"
     10 #include "util.h"
     11 
     12 #define EXPR_GEN_VALUE -1
     13 
     14 static const struct gen_value gv_void = {
     15 	.kind = GV_CONST,
     16 	.type = &builtin_type_void,
     17 };
     18 
     19 static struct gen_value gen_expr(struct gen_context *ctx,
     20 	const struct expression *expr);
     21 static void gen_expr_at(struct gen_context *ctx,
     22 	const struct expression *expr,
     23 	struct gen_value out);
     24 static struct gen_value gen_expr_with(struct gen_context *ctx,
     25 	const struct expression *expr,
     26 	struct gen_value *out);
     27 static void gen_global_decl(struct gen_context *ctx,
     28 	const struct declaration *decl);
     29 
     30 static void
     31 gen_defers(struct gen_context *ctx, struct gen_scope *scope)
     32 {
     33 	if (ctx->deferring || !scope) {
     34 		return;
     35 	}
     36 	ctx->deferring = true;
     37 	if (scope->defers) {
     38 		pushc(ctx->current, "gen defers");
     39 	}
     40 	for (struct gen_defer *defer = scope->defers; defer;
     41 			defer = defer->next) {
     42 		gen_expr(ctx, defer->expr);
     43 	}
     44 	ctx->deferring = false;
     45 }
     46 
     47 static struct gen_scope *
     48 gen_scope_lookup(struct gen_context *ctx, const struct scope *which)
     49 {
     50 	for (struct gen_scope *scope = ctx->scope;
     51 			scope; scope = scope->parent) {
     52 		if (scope->scope == which) {
     53 			return scope;
     54 		}
     55 	}
     56 	abort();
     57 }
     58 
     59 static struct gen_scope *
     60 push_scope(struct gen_context *ctx, const struct scope *scope)
     61 {
     62 	struct gen_scope *new = xcalloc(1, sizeof(struct gen_scope));
     63 	new->parent = ctx->scope;
     64 	new->scope = scope;
     65 	ctx->scope = new;
     66 	return new;
     67 }
     68 
     69 static void
     70 pop_scope(struct gen_context *ctx, bool gendefers)
     71 {
     72 	if (gendefers) {
     73 		gen_defers(ctx, ctx->scope);
     74 	}
     75 	struct gen_scope *scope = ctx->scope;
     76 	ctx->scope = scope->parent;
     77 	for (struct gen_defer *defer = scope->defers; defer; /* n/a */) {
     78 		struct gen_defer *next = defer->next;
     79 		free(defer);
     80 		defer = next;
     81 	}
     82 	free(scope);
     83 }
     84 
     85 static void
     86 gen_copy_memcpy(struct gen_context *ctx,
     87 	struct gen_value dest, struct gen_value src)
     88 {
     89 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memcpy");
     90 	struct qbe_value dtemp = mklval(ctx, &dest);
     91 	struct qbe_value stemp = mklval(ctx, &src);
     92 	struct qbe_value sz = constl(dest.type->size);
     93 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &dtemp, &stemp, &sz, NULL);
     94 }
     95 
     96 static void
     97 gen_copy_aligned(struct gen_context *ctx,
     98 	struct gen_value dest, struct gen_value src)
     99 {
    100 	if (dest.type->size > 128) {
    101 		gen_copy_memcpy(ctx, dest, src);
    102 		return;
    103 	}
    104 	struct qbe_value srcv = mkqval(ctx, &src);
    105 	struct qbe_value destv = mkqval(ctx, &dest);
    106 	struct qbe_value size = constl(dest.type->size);
    107 	pushi(ctx->current, NULL, Q_BLIT, &srcv, &destv, &size, NULL);
    108 }
    109 
    110 static void
    111 gen_store(struct gen_context *ctx,
    112 	struct gen_value object,
    113 	struct gen_value value)
    114 {
    115 	const struct type *ty = type_dealias(object.type);
    116 	switch (ty->storage) {
    117 	case STORAGE_ARRAY:
    118 	case STORAGE_SLICE:
    119 	case STORAGE_STRING:
    120 	case STORAGE_STRUCT:
    121 	case STORAGE_TAGGED:
    122 	case STORAGE_TUPLE:
    123 	case STORAGE_VALIST:
    124 		gen_copy_aligned(ctx, object, value);
    125 		return;
    126 	case STORAGE_UNION:
    127 		gen_copy_memcpy(ctx, object, value);
    128 		return;
    129 	case STORAGE_ENUM:
    130 		object.type = ty->alias.type;
    131 		break;
    132 	default:
    133 		break; // no-op
    134 	}
    135 
    136 	struct qbe_value qobj = mkqval(ctx, &object),
    137 		qval = mkqval(ctx, &value);
    138 	enum qbe_instr qi = store_for_type(ctx, object.type);
    139 	pushi(ctx->current, NULL, qi, &qval, &qobj, NULL);
    140 }
    141 
    142 static struct gen_value
    143 gen_load(struct gen_context *ctx, struct gen_value object)
    144 {
    145 	const struct type *ty = type_dealias(object.type);
    146 	switch (ty->storage) {
    147 	case STORAGE_ARRAY:
    148 	case STORAGE_FUNCTION:
    149 	case STORAGE_SLICE:
    150 	case STORAGE_STRING:
    151 	case STORAGE_STRUCT:
    152 	case STORAGE_TAGGED:
    153 	case STORAGE_TUPLE:
    154 	case STORAGE_UNION:
    155 	case STORAGE_VALIST:
    156 		return object;
    157 	case STORAGE_ENUM:
    158 		object.type = ty->alias.type;
    159 		break;
    160 	default:
    161 		break; // no-op
    162 	}
    163 
    164 	struct gen_value value = {
    165 		.kind = GV_TEMP,
    166 		.type = object.type,
    167 		.name = gen_name(ctx, "load.%d"),
    168 	};
    169 	struct qbe_value qobj = mkqval(ctx, &object),
    170 		qval = mkqval(ctx, &value);
    171 	enum qbe_instr qi = load_for_type(ctx, object.type);
    172 	pushi(ctx->current, &qval, qi, &qobj, NULL);
    173 	return value;
    174 }
    175 
    176 static void
    177 gen_fixed_abort(struct gen_context *ctx,
    178 	struct location loc, enum fixed_aborts reason)
    179 {
    180 	int n = snprintf(NULL, 0, "%s:%d:%d",
    181 			sources[loc.file], loc.lineno, loc.colno);
    182 	char *s = xcalloc(1, n + 1);
    183 	snprintf(s, n, "%s:%d:%d",
    184 			sources[loc.file], loc.lineno, loc.colno);
    185 	struct expression eloc = {0};
    186 	eloc.type = EXPR_CONSTANT;
    187 	eloc.result = &builtin_type_const_str;
    188 	eloc.constant.string.value = s;
    189 	eloc.constant.string.len = n - 1;
    190 	struct gen_value msg = gen_expr(ctx, &eloc);
    191 	struct qbe_value qmsg = mkqval(ctx, &msg);
    192 	struct qbe_value rtabort = mkrtfunc(ctx, "rt.abort_fixed");
    193 	struct qbe_value tmp = constl(reason);
    194 	pushi(ctx->current, NULL, Q_CALL, &rtabort, &qmsg, &tmp, NULL);
    195 }
    196 
    197 static struct gen_value
    198 gen_autoderef(struct gen_context *ctx, struct gen_value val)
    199 {
    200 	while (type_dealias(val.type)->storage == STORAGE_POINTER) {
    201 		val.type = type_dealias(val.type)->pointer.referent;
    202 		val = gen_load(ctx, val);
    203 	}
    204 	return val;
    205 }
    206 
    207 static struct gen_value
    208 gen_access_ident(struct gen_context *ctx, const struct scope_object *obj)
    209 {
    210 	switch (obj->otype) {
    211 	case O_BIND:
    212 		for (const struct gen_binding *gb = ctx->bindings;
    213 				gb; gb = gb->next) {
    214 			if (gb->object == obj) {
    215 				return gb->value;
    216 			}
    217 		}
    218 		break;
    219 	case O_DECL:
    220 		return (struct gen_value){
    221 			.kind = GV_GLOBAL,
    222 			.type = obj->type,
    223 			.name = ident_to_sym(&obj->ident),
    224 			.threadlocal = obj->threadlocal,
    225 		};
    226 	case O_CONST:
    227 	case O_TYPE:
    228 	case O_SCAN:
    229 		abort(); // Invariant
    230 	}
    231 	abort(); // Invariant
    232 }
    233 
    234 static struct gen_value
    235 gen_access_index(struct gen_context *ctx, const struct expression *expr)
    236 {
    237 	struct gen_value glval = gen_expr(ctx, expr->access.array);
    238 	glval = gen_autoderef(ctx, glval);
    239 	struct qbe_value qlval = mkqval(ctx, &glval);
    240 	struct qbe_value qival = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    241 	bool checkbounds = !expr->access.bounds_checked;
    242 	struct qbe_value length;
    243 	const struct type *ty = type_dealias(glval.type);
    244 	switch (ty->storage) {
    245 	case STORAGE_SLICE:;
    246 		enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    247 		struct qbe_value base = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    248 		pushi(ctx->current, &base, load, &qlval, NULL);
    249 
    250 		struct qbe_value temp = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    251 		length = mkqtmp(ctx, ctx->arch.sz, "len.%d");
    252 		struct qbe_value offset = constl(builtin_type_size.size);
    253 		pushi(ctx->current, &temp, Q_ADD, &qlval, &offset, NULL);
    254 		pushi(ctx->current, &length, load, &temp, NULL);
    255 
    256 		qlval = base;
    257 		break;
    258 	case STORAGE_ARRAY:
    259 		if (ty->array.length != SIZE_UNDEFINED) {
    260 			length = constl(ty->array.length);
    261 		} else {
    262 			checkbounds = false;
    263 		}
    264 		break;
    265 	default:
    266 		assert(0); // Unreachable
    267 	}
    268 
    269 	struct gen_value index = gen_expr(ctx, expr->access.index);
    270 	struct qbe_value qindex = mkqval(ctx, &index);
    271 	struct qbe_value itemsz = constl(expr->result->size);
    272 	pushi(ctx->current, &qival, Q_MUL, &qindex, &itemsz, NULL);
    273 	pushi(ctx->current, &qival, Q_ADD, &qlval, &qival, NULL);
    274 
    275 	if (checkbounds) {
    276 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
    277 		pushi(ctx->current, &valid, Q_CULTL, &qindex, &length, NULL);
    278 
    279 		struct qbe_statement linvalid, lvalid;
    280 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    281 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    282 
    283 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
    284 		push(&ctx->current->body, &linvalid);
    285 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
    286 		push(&ctx->current->body, &lvalid);
    287 	}
    288 
    289 	return (struct gen_value){
    290 		.kind = GV_TEMP,
    291 		.type = expr->result,
    292 		.name = qival.name,
    293 	};
    294 }
    295 
    296 static struct gen_value
    297 gen_access_field(struct gen_context *ctx, const struct expression *expr)
    298 {
    299 	const struct struct_field *field = expr->access.field;
    300 	struct gen_value glval = gen_expr(ctx, expr->access._struct);
    301 	glval = gen_autoderef(ctx, glval);
    302 	struct qbe_value qlval = mkqval(ctx, &glval);
    303 	struct qbe_value qfval = mkqtmp(ctx, ctx->arch.ptr, "field.%d");
    304 	struct qbe_value offs = constl(field->offset);
    305 	pushi(ctx->current, &qfval, Q_ADD, &qlval, &offs, NULL);
    306 	return (struct gen_value){
    307 		.kind = GV_TEMP,
    308 		.type = field->type,
    309 		.name = qfval.name,
    310 	};
    311 }
    312 
    313 static struct gen_value
    314 gen_access_value(struct gen_context *ctx, const struct expression *expr)
    315 {
    316 	const struct type_tuple *tuple = expr->access.tvalue;
    317 	struct gen_value glval = gen_expr(ctx, expr->access.tuple);
    318 	glval = gen_autoderef(ctx, glval);
    319 	struct qbe_value qlval = mkqval(ctx, &glval);
    320 	struct qbe_value qfval = mkqtmp(ctx, ctx->arch.ptr, "value.%d");
    321 	struct qbe_value offs = constl(tuple->offset);
    322 	pushi(ctx->current, &qfval, Q_ADD, &qlval, &offs, NULL);
    323 	return (struct gen_value){
    324 		.kind = GV_TEMP,
    325 		.type = tuple->type,
    326 		.name = qfval.name,
    327 	};
    328 }
    329 
    330 static struct gen_value
    331 gen_expr_access_addr(struct gen_context *ctx, const struct expression *expr)
    332 {
    333 	struct gen_value addr;
    334 	switch (expr->access.type) {
    335 	case ACCESS_IDENTIFIER:
    336 		addr = gen_access_ident(ctx, expr->access.object);
    337 		break;
    338 	case ACCESS_INDEX:
    339 		addr = gen_access_index(ctx, expr);
    340 		break;
    341 	case ACCESS_FIELD:
    342 		addr = gen_access_field(ctx, expr);
    343 		break;
    344 	case ACCESS_TUPLE:
    345 		addr = gen_access_value(ctx, expr);
    346 		break;
    347 	}
    348 	return addr;
    349 }
    350 
    351 static struct gen_value
    352 gen_expr_access(struct gen_context *ctx, const struct expression *expr)
    353 {
    354 	struct gen_value addr = gen_expr_access_addr(ctx, expr);
    355 	return gen_load(ctx, addr);
    356 }
    357 
    358 static void
    359 gen_alloc_slice_at(struct gen_context *ctx,
    360 		const struct expression *expr,
    361 		struct gen_value out,
    362 		bool expand)
    363 {
    364 	struct qbe_value qcap;
    365 	if (expr->alloc.cap) {
    366 		struct gen_value cap = gen_expr(ctx, expr->alloc.cap);
    367 		qcap = mkqval(ctx, &cap);
    368 	}
    369 
    370 	struct gen_value init;
    371 	struct qbe_value qinit;
    372 	struct qbe_value length, initdata;
    373 	const struct type *inittype = type_dealias(expr->alloc.init->result);
    374 	switch (inittype->storage) {
    375 	case STORAGE_ARRAY:
    376 		assert(inittype->array.length != SIZE_UNDEFINED);
    377 		length = constl(inittype->array.length);
    378 		break;
    379 	case STORAGE_SLICE:
    380 		init = gen_expr(ctx, expr->alloc.init);
    381 		qinit = mkqval(ctx, &init);
    382 		enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    383 		initdata = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    384 		pushi(ctx->current, &initdata, load, &qinit, NULL);
    385 		struct qbe_value offset = constl(builtin_type_size.size);
    386 		struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    387 		pushi(ctx->current, &ptr, Q_ADD, &qinit, &offset, NULL);
    388 		length = mkqtmp(ctx, ctx->arch.sz, ".%d");
    389 		pushi(ctx->current, &length, load, &ptr, NULL);
    390 		break;
    391 	default: abort(); // Invariant
    392 	}
    393 
    394 	if (!expr->alloc.cap) {
    395 		qcap = length;
    396 	}
    397 
    398 	const struct type *sltype = type_dealias(expr->result);
    399 	struct qbe_value isize = constl(sltype->array.members->size);
    400 	struct qbe_value size = mkqtmp(ctx, ctx->arch.sz, ".%d");
    401 	pushi(ctx->current, &size, Q_MUL, &qcap, &isize, NULL);
    402 
    403 	struct qbe_statement lzero, lnonzero;
    404 	struct qbe_value bzero = mklabel(ctx, &lzero, ".%d");
    405 	struct qbe_value bnonzero = mklabel(ctx, &lnonzero, ".%d");
    406 
    407 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.malloc");
    408 	struct qbe_value data = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    409 	struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
    410 	struct qbe_value zero = constl(0);
    411 	pushi(ctx->current, &data, Q_COPY, &zero, NULL);
    412 	pushi(ctx->current, &cmpres, Q_CNEL, &size, &zero, NULL);
    413 	pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bnonzero, &bzero, NULL);
    414 	push(&ctx->current->body, &lnonzero);
    415 	pushi(ctx->current, &data, Q_CALL, &rtfunc, &size, NULL);
    416 
    417 	struct qbe_statement linvalid;
    418 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    419 	pushi(ctx->current, &cmpres, Q_CNEL, &data, &zero, NULL);
    420 	pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bzero, &binvalid, NULL);
    421 	push(&ctx->current->body, &linvalid);
    422 	gen_fixed_abort(ctx, expr->loc, ABORT_ALLOC_FAILURE);
    423 	push(&ctx->current->body, &lzero);
    424 
    425 	struct qbe_value base = mklval(ctx, &out);
    426 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    427 	struct qbe_value offset = constl(builtin_type_size.size);
    428 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
    429 	pushi(ctx->current, NULL, store, &data, &base, NULL);
    430 	pushi(ctx->current, &ptr, Q_ADD, &base, &offset, NULL);
    431 
    432 	if (expand) {
    433 		pushi(ctx->current, NULL, store, &qcap, &ptr, NULL);
    434 	} else {
    435 		pushi(ctx->current, NULL, store, &length, &ptr, NULL);
    436 	}
    437 
    438 	offset = constl(builtin_type_size.size * 2);
    439 	pushi(ctx->current, &ptr, Q_ADD, &base, &offset, NULL);
    440 	pushi(ctx->current, NULL, store, &qcap, &ptr, NULL);
    441 
    442 	if (inittype->storage == STORAGE_ARRAY) {
    443 		struct gen_value storage = (struct gen_value){
    444 			.kind = GV_TEMP,
    445 			.type = inittype,
    446 			.name = data.name,
    447 		};
    448 		gen_expr_at(ctx, expr->alloc.init, storage);
    449 	} else {
    450 		struct qbe_value copysize = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    451 		pushi(ctx->current, &copysize, Q_MUL, &length, &isize, NULL);
    452 		struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    453 		pushi(ctx->current, NULL, Q_CALL, &rtmemcpy,
    454 				&data, &initdata, &copysize, NULL);
    455 	}
    456 
    457 	if (!expand) {
    458 		return;
    459 	}
    460 
    461 	struct qbe_value last = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    462 	struct qbe_value next = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    463 	pushi(ctx->current, &next, Q_MUL, &length, &isize, NULL);
    464 	pushi(ctx->current, &next, Q_ADD, &next, &data, NULL);
    465 	pushi(ctx->current, &last, Q_SUB, &next, &isize, NULL);
    466 	struct qbe_value remain = mkqtmp(ctx, ctx->arch.sz, ".%d");
    467 	pushi(ctx->current, &remain, Q_SUB, &qcap, &length, NULL);
    468 	pushi(ctx->current, &remain, Q_MUL, &remain, &isize, NULL);
    469 	struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    470 	pushi(ctx->current, NULL, Q_CALL, &rtmemcpy, &next, &last, &remain, NULL);
    471 }
    472 
    473 static struct gen_value
    474 gen_expr_alloc_init_with(struct gen_context *ctx,
    475 	const struct expression *expr, struct gen_value *out)
    476 {
    477 	// alloc(init) case
    478 	assert(expr->alloc.cap == NULL);
    479 
    480 	const struct type *objtype = type_dealias(expr->result);
    481 	assert(objtype->storage == STORAGE_POINTER);
    482 	objtype = objtype->pointer.referent;
    483 
    484 	struct qbe_value sz = constl(objtype->size);
    485 	struct gen_value result = mkgtemp(ctx, expr->result, ".%d");
    486 	struct qbe_value qresult = mkqval(ctx, &result);
    487 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.malloc");
    488 	pushi(ctx->current, &qresult, Q_CALL, &rtfunc, &sz, NULL);
    489 
    490 	if (!(type_dealias(expr->result)->pointer.flags & PTR_NULLABLE)) {
    491 		struct qbe_statement linvalid, lvalid;
    492 		struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
    493 		struct qbe_value zero = constl(0);
    494 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    495 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    496 
    497 		pushi(ctx->current, &cmpres, Q_CNEL, &qresult, &zero, NULL);
    498 		pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bvalid, &binvalid, NULL);
    499 		push(&ctx->current->body, &linvalid);
    500 		gen_fixed_abort(ctx, expr->loc, ABORT_ALLOC_FAILURE);
    501 		push(&ctx->current->body, &lvalid);
    502 	}
    503 
    504 	struct gen_value object = {
    505 		.kind = GV_TEMP,
    506 		.type = objtype,
    507 		.name = result.name,
    508 	};
    509 	gen_expr_at(ctx, expr->alloc.init, object);
    510 	if (out) {
    511 		gen_store(ctx, *out, result);
    512 	}
    513 	return result;
    514 }
    515 
    516 static struct gen_value
    517 gen_expr_alloc_slice_with(struct gen_context *ctx,
    518 	const struct expression *expr, struct gen_value *out)
    519 {
    520 	// alloc(init, cap | len) case
    521 	bool expand = expr->alloc.kind == ALLOC_WITH_LEN;
    522 	if (out) {
    523 		gen_alloc_slice_at(ctx, expr, *out, expand);
    524 		return gv_void;
    525 	}
    526 	struct gen_value temp = mkgtemp(ctx, expr->result, "object.%d");
    527 	struct qbe_value base = mkqval(ctx, &temp);
    528 	struct qbe_value sz = constl(expr->result->size);
    529 	enum qbe_instr alloc = alloc_for_align(expr->result->align);
    530 	pushprei(ctx->current, &base, alloc, &sz, NULL);
    531 	gen_alloc_slice_at(ctx, expr, temp, expand);
    532 	return temp;
    533 }
    534 
    535 static struct gen_value
    536 gen_expr_alloc_copy_with(struct gen_context *ctx,
    537 	const struct expression *expr, struct gen_value *out)
    538 {
    539 	// alloc(init...) case
    540 	assert(expr->alloc.cap == NULL);
    541 
    542 	struct gen_value ret = gv_void;
    543 	if (out == NULL) {
    544 		ret = mkgtemp(ctx, expr->result, "object.%d");
    545 		out = &ret;
    546 
    547 		struct qbe_value base = mkqval(ctx, out);
    548 		struct qbe_value sz = constl(expr->result->size);
    549 		enum qbe_instr alloc = alloc_for_align(expr->result->align);
    550 		pushprei(ctx->current, &base, alloc, &sz, NULL);
    551 	}
    552 
    553 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    554 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
    555 
    556 	struct gen_value src = gen_expr(ctx, expr->alloc.init);
    557 	struct qbe_value dbase = mkcopy(ctx, out, ".%d");
    558 	struct qbe_value offs = constl(builtin_type_size.size);
    559 
    560 	const struct type *initres = type_dealias(expr->alloc.init->result);
    561 	struct qbe_value srcdata;
    562 	struct qbe_value length;
    563 	if (initres->storage == STORAGE_SLICE) {
    564 		assert(initres->array.length == SIZE_UNDEFINED);
    565 		srcdata = mkqtmp(ctx, ctx->arch.sz, ".%d");
    566 		struct qbe_value sbase = mkcopy(ctx, &src, ".%d");
    567 		pushi(ctx->current, &srcdata, load, &sbase, NULL);
    568 		pushi(ctx->current, &sbase, Q_ADD, &sbase, &offs, NULL);
    569 		length = mkqtmp(ctx, ctx->arch.sz, ".%d");
    570 		pushi(ctx->current, &length, load, &sbase, NULL);
    571 	} else if (initres->storage == STORAGE_ARRAY) {
    572 		assert(initres->array.length != SIZE_UNDEFINED);
    573 		srcdata = mkcopy(ctx, &src, ".%d"); // TODO: object.%d
    574 		length = constl(initres->array.length);
    575 	} else {
    576 		abort();
    577 	}
    578 
    579 	struct qbe_statement linvalid, lvalid, lalloc, lcopy;
    580 	struct qbe_value balloc = mklabel(ctx, &lalloc, ".%d");
    581 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    582 	struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    583 	struct qbe_value bcopy = mklabel(ctx, &lcopy, ".%d");
    584 
    585 	struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
    586 	struct qbe_value newdata = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    587 	struct qbe_value zero = constl(0);
    588 	pushi(ctx->current, &newdata, Q_COPY, &zero, NULL);
    589 	pushi(ctx->current, &cmpres, Q_CNEL, &length, &zero, NULL);
    590 	pushi(ctx->current, NULL, Q_JNZ, &cmpres, &balloc, &bvalid, NULL);
    591 	push(&ctx->current->body, &lalloc);
    592 
    593 	const struct type *result = type_dealias(expr->result);
    594 	assert(result->storage == STORAGE_SLICE);
    595 	struct qbe_value sz = mkqtmp(ctx, ctx->arch.sz, ".%d");
    596 	struct qbe_value membsz = constl(result->array.members->size);
    597 	pushi(ctx->current, &sz, Q_MUL, &membsz, &length, NULL);
    598 
    599 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.malloc");
    600 	pushi(ctx->current, &newdata, Q_CALL, &rtfunc, &sz, NULL);
    601 	pushi(ctx->current, &cmpres, Q_CNEL, &newdata, &zero, NULL);
    602 	pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bcopy, &binvalid, NULL);
    603 
    604 	push(&ctx->current->body, &linvalid);
    605 	gen_fixed_abort(ctx, expr->loc, ABORT_ALLOC_FAILURE);
    606 
    607 	push(&ctx->current->body, &lcopy);
    608 	struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    609 	pushi(ctx->current, NULL, Q_CALL, &rtmemcpy, &newdata, &srcdata, &sz, NULL);
    610 
    611 	push(&ctx->current->body, &lvalid);
    612 	pushi(ctx->current, NULL, store, &newdata, &dbase, NULL);
    613 	pushi(ctx->current, &dbase, Q_ADD, &dbase, &offs, NULL);
    614 	pushi(ctx->current, NULL, store, &length, &dbase, NULL);
    615 	pushi(ctx->current, &dbase, Q_ADD, &dbase, &offs, NULL);
    616 	pushi(ctx->current, NULL, store, &length, &dbase, NULL);
    617 
    618 	return ret;
    619 }
    620 
    621 static struct gen_value
    622 gen_expr_alloc_with(struct gen_context *ctx,
    623 	const struct expression *expr, struct gen_value *out)
    624 {
    625 	switch (expr->alloc.kind) {
    626 	case ALLOC_OBJECT:
    627 		return gen_expr_alloc_init_with(ctx, expr, out);
    628 	case ALLOC_WITH_CAP:
    629 	case ALLOC_WITH_LEN:
    630 		return gen_expr_alloc_slice_with(ctx, expr, out);
    631 	case ALLOC_COPY:
    632 		return gen_expr_alloc_copy_with(ctx, expr, out);
    633 	}
    634 	abort(); // Unreachable
    635 }
    636 
    637 static struct gen_value
    638 gen_expr_append(struct gen_context *ctx, const struct expression *expr)
    639 {
    640 	struct gen_value slice = gen_expr(ctx, expr->append.object);
    641 	slice = gen_autoderef(ctx, slice);
    642 	struct qbe_value qslice = mkqval(ctx, &slice);
    643 
    644 	struct qbe_value lenptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    645 	struct qbe_value prevlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    646 	struct qbe_value offs = constl(builtin_type_size.size);
    647 	pushi(ctx->current, &lenptr, Q_ADD, &qslice, &offs, NULL);
    648 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
    649 	pushi(ctx->current, &prevlen, load, &lenptr, NULL);
    650 
    651 	struct qbe_value appendlen;
    652 	const struct type *valtype = type_dealias(expr->append.value->result);
    653 	if (expr->append.length != NULL) {
    654 		struct gen_value length = gen_expr(ctx, expr->append.length);
    655 		appendlen = mkqval(ctx, &length);
    656 		assert(valtype->storage == STORAGE_ARRAY && valtype->array.expandable);
    657 	} else if (!expr->append.is_multi) {
    658 		appendlen = constl(1);
    659 	}
    660 
    661 	struct gen_value value;
    662 	struct qbe_value qvalue;
    663 	if (!expr->append.is_multi || valtype->storage != STORAGE_ARRAY) {
    664 		// We use gen_expr_at for the array case to avoid a copy
    665 		value = gen_expr(ctx, expr->append.value);
    666 		qvalue = mkqval(ctx, &value);
    667 	}
    668 
    669 	if (expr->append.is_multi) {
    670 		if (valtype->storage == STORAGE_ARRAY) {
    671 			assert(valtype->array.length != SIZE_UNDEFINED);
    672 			appendlen = constl(valtype->array.length);
    673 		} else {
    674 			appendlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    675 			struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    676 			offs = constl(builtin_type_size.size);
    677 			pushi(ctx->current, &ptr, Q_ADD, &qvalue, &offs, NULL);
    678 			pushi(ctx->current, &appendlen, load, &ptr, NULL);
    679 		}
    680 	}
    681 
    682 	struct qbe_value newlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    683 	pushi(ctx->current, &newlen, Q_ADD, &prevlen, &appendlen, NULL);
    684 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
    685 	pushi(ctx->current, NULL, store, &newlen, &lenptr, NULL);
    686 
    687 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    688 	const struct type *mtype = type_dealias(slice.type)->array.members;
    689 	struct qbe_value membsz = constl(mtype->size);
    690 	if (!expr->append.is_static) {
    691 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.ensure");
    692 		struct qbe_value lval = mklval(ctx, &slice);
    693 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &lval, &membsz, NULL);
    694 	} else {
    695 		offs = constl(builtin_type_size.size * 2);
    696 		pushi(ctx->current, &ptr, Q_ADD, &qslice, &offs, NULL);
    697 		struct qbe_value cap = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    698 		pushi(ctx->current, &cap, load, &ptr, NULL);
    699 
    700 		struct qbe_statement lvalid, linvalid;
    701 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    702 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    703 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
    704 		pushi(ctx->current, &valid, Q_CULEL, &newlen, &cap, NULL);
    705 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
    706 
    707 		push(&ctx->current->body, &linvalid);
    708 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
    709 		push(&ctx->current->body, &lvalid);
    710 	}
    711 
    712 	offs = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    713 	pushi(ctx->current, &ptr, load, &qslice, NULL);
    714 	pushi(ctx->current, &offs, Q_MUL, &prevlen, &membsz, NULL);
    715 	pushi(ctx->current, &ptr, Q_ADD, &ptr, &offs, NULL);
    716 
    717 	struct gen_value item = {
    718 		.kind = GV_TEMP,
    719 		.type = mtype,
    720 		.name = ptr.name,
    721 	};
    722 	if (expr->append.is_multi && valtype->storage == STORAGE_ARRAY) {
    723 		item.type = valtype;
    724 		gen_expr_at(ctx, expr->append.value, item);
    725 	} else if (expr->append.is_multi && valtype->storage == STORAGE_SLICE) {
    726 		struct qbe_value qsrc = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    727 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memmove");
    728 		struct qbe_value sz = mkqtmp(ctx, ctx->arch.sz, ".%d");
    729 		pushi(ctx->current, &sz, Q_MUL, &appendlen, &membsz, NULL);
    730 		pushi(ctx->current, &qsrc, load, &qvalue, NULL);
    731 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &ptr, &qsrc, &sz, NULL);
    732 	} else if (expr->append.length != NULL) {
    733 		// XXX: This could be made more efficient for some cases if
    734 		// check could determine the length at compile time and lower it
    735 		// to a fixed-length array type
    736 		assert(valtype->storage == STORAGE_ARRAY);
    737 		item.type = valtype;
    738 		gen_expr_at(ctx, expr->append.value, item);
    739 
    740 		assert(valtype->array.length != SIZE_UNDEFINED);
    741 		struct qbe_value next = mkqtmp(ctx, ctx->arch.ptr, "next.%d");
    742 		struct qbe_value last = mkqtmp(ctx, ctx->arch.ptr, "last.%d");
    743 		struct qbe_value arlen = constl(valtype->array.length * mtype->size);
    744 		pushi(ctx->current, &next, Q_ADD, &ptr, &arlen, NULL);
    745 		arlen = constl((valtype->array.length - 1) * mtype->size);
    746 		pushi(ctx->current, &last, Q_ADD, &ptr, &arlen, NULL);
    747 
    748 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memcpy");
    749 		struct qbe_value remain = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    750 		struct qbe_value one = constl(1);
    751 		pushi(ctx->current, &remain, Q_SUB, &appendlen, &one, NULL);
    752 		pushi(ctx->current, &remain, Q_MUL, &remain, &membsz, NULL);
    753 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &next, &last, &remain, NULL);
    754 	} else {
    755 		gen_store(ctx, item, value);
    756 	}
    757 
    758 	return gv_void;
    759 }
    760 
    761 static struct gen_value
    762 gen_expr_assert(struct gen_context *ctx, const struct expression *expr)
    763 {
    764 	assert(expr->assert.message); // Invariant
    765 	if (expr->assert.is_static) {
    766 		return gv_void;
    767 	}
    768 
    769 	struct gen_value msg;
    770 	struct qbe_statement failedl, passedl;
    771 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.abort");
    772 	if (expr->assert.cond) {
    773 		struct qbe_value bfailed = mklabel(ctx, &failedl, "failed.%d");
    774 		struct qbe_value bpassed = mklabel(ctx, &passedl, "passed.%d");
    775 		struct gen_value cond = gen_expr(ctx, expr->assert.cond);
    776 		struct qbe_value qcond = mkqval(ctx, &cond);
    777 		pushi(ctx->current, NULL, Q_JNZ, &qcond, &bpassed, &bfailed, NULL);
    778 		push(&ctx->current->body, &failedl);
    779 		msg = gen_expr(ctx, expr->assert.message);
    780 	} else {
    781 		msg = gen_expr(ctx, expr->assert.message);
    782 	}
    783 
    784 	for (struct gen_scope *scope = ctx->scope; scope; scope = scope->parent) {
    785 		gen_defers(ctx, scope);
    786 	}
    787 
    788 	struct qbe_value qmsg = mkqval(ctx, &msg);
    789 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &qmsg, NULL);
    790 
    791 	if (expr->assert.cond) {
    792 		push(&ctx->current->body, &passedl);
    793 	}
    794 
    795 	return gv_void;
    796 }
    797 
    798 static struct gen_value
    799 gen_expr_assign_slice_expandable(struct gen_context *ctx, const struct expression *expr)
    800 {
    801 	struct gen_value obj = gen_expr(ctx, expr->assign.object);
    802 	struct qbe_value qobj = mkqval(ctx, &obj);
    803 	
    804 	// get the length of the copy
    805 	struct qbe_value step = constl(ctx->arch.ptr->size);
    806 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    807 	struct qbe_value olen = mkqtmp(ctx, ctx->arch.sz, ".%d");
    808 	pushi(ctx->current, &ptr, Q_ADD, &qobj, &step, NULL);
    809 	pushi(ctx->current, &olen, Q_LOADL, &ptr, NULL);
    810 	
    811 	// check if there is anything to do
    812 	struct qbe_statement lzero, lnonzero;
    813 	struct qbe_value bzero = mklabel(ctx, &lzero, ".%d");
    814 	struct qbe_value bnonzero = mklabel(ctx, &lnonzero, ".%d");
    815 	
    816 	struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
    817 	struct qbe_value zero = constl(0);
    818 	pushi(ctx->current, &cmpres, Q_CNEL, &olen, &zero, NULL);
    819 	pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bnonzero, &bzero, NULL);
    820 	push(&ctx->current->body, &lnonzero);
    821 	
    822 	// get the destination
    823 	struct qbe_value odata = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    824 	pushi(ctx->current, &odata, Q_LOADL, &qobj, NULL);
    825 	
    826 	// get the source
    827 	struct gen_value val = gen_expr(ctx, expr->assign.value);
    828 	struct qbe_value qval = mkqval(ctx, &val);
    829 	struct qbe_value vdata = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    830 	pushi(ctx->current, &vdata, Q_LOADL, &qval, NULL);
    831 	
    832 	// copy the first item
    833 	const struct type *sltype = expr->assign.object->result->array.members;
    834 	enum qbe_instr store = store_for_type(ctx, sltype);
    835 	pushi(ctx->current, NULL, store, &vdata, &odata, NULL);
    836 	
    837 	// perform the copy minus the first element
    838 	struct qbe_value isize = constl(sltype->size);
    839 	struct qbe_value next = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    840 	pushi(ctx->current, &next, Q_ADD, &odata, &isize, NULL);
    841 	struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
    842 	struct qbe_value one = constl(1);
    843 	pushi(ctx->current, &olen, Q_SUB, &olen, &one, NULL);
    844 	pushi(ctx->current, &olen, Q_MUL, &olen, &isize, NULL);
    845 	pushi(ctx->current, NULL, Q_CALL, &rtmemcpy, &next, &odata, &olen, NULL);
    846 	
    847 	push(&ctx->current->body, &lzero);
    848 	
    849 	return gv_void;
    850 }
    851 
    852 static struct gen_value
    853 gen_expr_assign_slice(struct gen_context *ctx, const struct expression *expr)
    854 {
    855 	const struct type *vtype = type_dealias(expr->assign.value->result);
    856 	if (vtype->storage == STORAGE_ARRAY && vtype->array.expandable) {
    857 		return gen_expr_assign_slice_expandable(ctx, expr);
    858 	}
    859 
    860 	struct gen_value obj = gen_expr(ctx, expr->assign.object);
    861 	struct gen_value val = gen_expr(ctx, expr->assign.value);
    862 	struct qbe_value qobj = mkqval(ctx, &obj);
    863 	struct qbe_value qval = mkqval(ctx, &val);
    864 	struct qbe_value step = constl(ctx->arch.ptr->size);
    865 
    866 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    867 	struct qbe_value olen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    868 	struct qbe_value vlen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    869 
    870 	pushi(ctx->current, &ptr, Q_ADD, &qobj, &step, NULL);
    871 	pushi(ctx->current, &olen, Q_LOADL, &ptr, NULL);
    872 	pushi(ctx->current, &ptr, Q_ADD, &qval, &step, NULL);
    873 	pushi(ctx->current, &vlen, Q_LOADL, &ptr, NULL);
    874 
    875 	struct qbe_statement linvalid, lvalid;
    876 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
    877 	struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
    878 	struct qbe_value tmp = mkqtmp(ctx, &qbe_long, ".%d");
    879 	pushi(ctx->current, &tmp, Q_CEQL, &olen, &vlen, NULL);
    880 	pushi(ctx->current, NULL, Q_JNZ, &tmp, &bvalid, &binvalid, NULL);
    881 	push(&ctx->current->body, &linvalid);
    882 	gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
    883 	push(&ctx->current->body, &lvalid);
    884 
    885 	struct qbe_value rtmemmove = mkrtfunc(ctx, "rt.memmove");
    886 	struct qbe_value optr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    887 	struct qbe_value vptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
    888 	pushi(ctx->current, &optr, Q_LOADL, &qobj, NULL);
    889 	pushi(ctx->current, &vptr, Q_LOADL, &qval, NULL);
    890 	tmp = constl(expr->assign.object->result->array.members->size);
    891 	pushi(ctx->current, &olen, Q_MUL, &olen, &tmp, NULL);
    892 	pushi(ctx->current, NULL, Q_CALL, &rtmemmove, &optr, &vptr, &olen, NULL);
    893 
    894 	return gv_void;
    895 }
    896 
    897 static struct gen_value
    898 gen_expr_assign(struct gen_context *ctx, const struct expression *expr)
    899 {
    900 	struct expression *object = expr->assign.object;
    901 	struct expression *value = expr->assign.value;
    902 	if (object->type == EXPR_SLICE) {
    903 		return gen_expr_assign_slice(ctx, expr);
    904 	}
    905 	assert(object->type == EXPR_ACCESS || expr->assign.indirect); // Invariant
    906 
    907 	struct gen_value obj;
    908 	if (expr->assign.indirect) {
    909 		obj = gen_expr(ctx, object);
    910 		obj.type = type_dealias(object->result)->pointer.referent;
    911 	} else {
    912 		obj = gen_expr_access_addr(ctx, object);
    913 	}
    914 	if (expr->assign.op == BIN_LEQUAL) {
    915 		gen_expr_at(ctx, value, obj);
    916 	} else if (expr->assign.op == BIN_LAND || expr->assign.op == BIN_LOR) {
    917 		struct qbe_statement lrval, lshort;
    918 		struct qbe_value brval = mklabel(ctx, &lrval, ".%d");
    919 		struct qbe_value bshort = mklabel(ctx, &lshort, ".%d");
    920 		struct gen_value load = gen_load(ctx, obj);
    921 		struct qbe_value qload = mkqval(ctx, &load);
    922 		if (expr->binarithm.op == BIN_LAND) {
    923 			pushi(ctx->current, NULL, Q_JNZ, &qload, &brval,
    924 				&bshort, NULL);
    925 		} else {
    926 			pushi(ctx->current, NULL, Q_JNZ, &qload, &bshort,
    927 				&brval, NULL);
    928 		}
    929 		push(&ctx->current->body, &lrval);
    930 		gen_expr_at(ctx, value, obj);
    931 		if (!expr->binarithm.rvalue->terminates) {
    932 			pushi(ctx->current, NULL, Q_JMP, &bshort, NULL);
    933 		}
    934 		push(&ctx->current->body, &lshort);
    935 	} else {
    936 		struct gen_value lvalue = gen_load(ctx, obj);
    937 		struct gen_value rvalue = gen_expr(ctx, value);
    938 		struct qbe_value qlval = mkqval(ctx, &lvalue);
    939 		struct qbe_value qrval = mkqval(ctx, &rvalue);
    940 		enum qbe_instr instr = binarithm_for_op(ctx,
    941 			expr->assign.op, lvalue.type);
    942 		pushi(ctx->current, &qlval, instr, &qlval, &qrval, NULL);
    943 		gen_store(ctx, obj, lvalue);
    944 	}
    945 
    946 	return gv_void;
    947 }
    948 
    949 static struct qbe_value
    950 extend(struct gen_context *ctx, struct qbe_value v, const struct type *type)
    951 {
    952 	enum qbe_instr op;
    953 	switch (type->size) {
    954 	case 1:
    955 		op = type_is_signed(type) ? Q_EXTSB : Q_EXTUB;
    956 		break;
    957 	case 2:
    958 		op = type_is_signed(type) ? Q_EXTSH : Q_EXTUH;
    959 		break;
    960 	default:
    961 		return v;
    962 	}
    963 
    964 	struct qbe_value temp = mkqtmp(ctx, &qbe_word, "ext.%d");
    965 	pushi(ctx->current, &temp, op, &v, NULL);
    966 	return temp;
    967 }
    968 
    969 static struct gen_value
    970 gen_expr_binarithm(struct gen_context *ctx, const struct expression *expr)
    971 {
    972 	const struct type *ltype = type_dealias(expr->binarithm.lvalue->result);
    973 	const struct type *rtype = type_dealias(expr->binarithm.rvalue->result);
    974 	struct gen_value result = mkgtemp(ctx, expr->result, ".%d");
    975 	struct qbe_value qresult = mkqval(ctx, &result);
    976 
    977 	if (expr->binarithm.op == BIN_LAND || expr->binarithm.op == BIN_LOR) {
    978 		struct qbe_statement lrval, lshort;
    979 		struct qbe_value brval = mklabel(ctx, &lrval, ".%d");
    980 		struct qbe_value bshort = mklabel(ctx, &lshort, ".%d");
    981 		struct gen_value lval = gen_expr(ctx, expr->binarithm.lvalue);
    982 		struct qbe_value qlval = mkqval(ctx, &lval);
    983 		pushi(ctx->current, &qresult, Q_COPY, &qlval, NULL);
    984 		if (expr->binarithm.op == BIN_LAND) {
    985 			pushi(ctx->current, NULL, Q_JNZ, &qresult, &brval,
    986 				&bshort, NULL);
    987 		} else {
    988 			pushi(ctx->current, NULL, Q_JNZ, &qresult, &bshort,
    989 				&brval, NULL);
    990 		}
    991 		push(&ctx->current->body, &lrval);
    992 		struct gen_value rval = gen_expr(ctx, expr->binarithm.rvalue);
    993 		struct qbe_value qrval = mkqval(ctx, &rval);
    994 		pushi(ctx->current, &qresult, Q_COPY, &qrval, NULL);
    995 		if (!expr->binarithm.rvalue->terminates) {
    996 			pushi(ctx->current, NULL, Q_JMP, &bshort, NULL);
    997 		}
    998 		push(&ctx->current->body, &lshort);
    999 		return result;
   1000 	}
   1001 
   1002 	struct gen_value lvalue = gen_expr(ctx, expr->binarithm.lvalue);
   1003 	struct gen_value rvalue = gen_expr(ctx, expr->binarithm.rvalue);
   1004 	struct qbe_value qlval = mkqval(ctx, &lvalue);
   1005 	struct qbe_value qrval = mkqval(ctx, &rvalue);
   1006 
   1007 	switch (expr->binarithm.op) {
   1008 	case BIN_GREATER:
   1009 	case BIN_GREATEREQ:
   1010 	case BIN_LEQUAL:
   1011 	case BIN_LESS:
   1012 	case BIN_LESSEQ:
   1013 	case BIN_NEQUAL:
   1014 		qlval = extend(ctx, qlval, ltype);
   1015 		qrval = extend(ctx, qrval, rtype);
   1016 		break;
   1017 	default:
   1018 		break;
   1019 	}
   1020 
   1021 	assert((ltype->storage == STORAGE_STRING) == (rtype->storage == STORAGE_STRING));
   1022 	if (ltype->storage == STORAGE_STRING) {
   1023 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.strcmp");
   1024 		pushi(ctx->current, &qresult, Q_CALL,
   1025 			&rtfunc, &qlval, &qrval, NULL);
   1026 		if (expr->binarithm.op == BIN_NEQUAL) {
   1027 			struct qbe_value one = constl(1);
   1028 			pushi(ctx->current, &qresult, Q_XOR, &qresult, &one, NULL);
   1029 		} else {
   1030 			assert(expr->binarithm.op == BIN_LEQUAL);
   1031 		}
   1032 		return result;
   1033 	}
   1034 	enum qbe_instr instr = binarithm_for_op(ctx, expr->binarithm.op,
   1035 		expr->binarithm.lvalue->result);
   1036 	pushi(ctx->current, &qresult, instr, &qlval, &qrval, NULL);
   1037 	return result;
   1038 }
   1039 
   1040 static void
   1041 gen_expr_binding_unpack_static(struct gen_context *ctx,
   1042 	const struct expression_binding *binding)
   1043 {
   1044 	assert(binding->object == NULL);
   1045 
   1046 	struct tuple_constant *tupleconst =
   1047 		binding->initializer->constant.tuple;
   1048 
   1049 	for (const struct binding_unpack *unpack = binding->unpack;
   1050 			unpack; unpack = unpack->next) {
   1051 		if (unpack->object == NULL) {
   1052 			goto done;
   1053 		}
   1054 		assert(unpack->object->otype == O_DECL);
   1055 
   1056 		struct declaration decl = {
   1057 			.type = DECL_GLOBAL,
   1058 			.ident = unpack->object->ident,
   1059 			.global = {
   1060 				.type = unpack->object->type,
   1061 				.value = tupleconst->value,
   1062 			},
   1063 		};
   1064 		gen_global_decl(ctx, &decl);
   1065 
   1066 done:
   1067 		tupleconst = tupleconst->next;
   1068 	}
   1069 }
   1070 
   1071 static void
   1072 gen_expr_binding_unpack(struct gen_context *ctx,
   1073 	const struct expression_binding *binding)
   1074 {
   1075 	assert(binding->object == NULL);
   1076 
   1077 	const struct type *type = binding->initializer->result;
   1078 	char *tuple_name = gen_name(ctx, "tupleunpack.%d");
   1079 	struct gen_value tuple_gv = {
   1080 		.kind = GV_TEMP,
   1081 		.type = type,
   1082 		.name = tuple_name,
   1083 	};
   1084 	struct qbe_value tuple_qv = mklval(ctx, &tuple_gv);
   1085 	struct qbe_value sz = constl(type->size);
   1086 	enum qbe_instr alloc = alloc_for_align(type->align);
   1087 	pushprei(ctx->current, &tuple_qv, alloc, &sz, NULL);
   1088 
   1089 	gen_expr_at(ctx, binding->initializer, tuple_gv);
   1090 
   1091 	for (const struct binding_unpack *unpack = binding->unpack;
   1092 			unpack; unpack = unpack->next) {
   1093 		if (unpack->object == NULL) {
   1094 			continue;
   1095 		}
   1096 		assert(unpack->object->otype != O_DECL);
   1097 
   1098 		const struct type *type = unpack->object->type;
   1099 		struct gen_value item_gv = {
   1100 			.kind = GV_TEMP,
   1101 			.type = type,
   1102 			.name = gen_name(ctx, "binding.%d"),
   1103 		};
   1104 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   1105 		gb->value = item_gv;
   1106 		gb->object = unpack->object;
   1107 		gb->next = ctx->bindings;
   1108 		ctx->bindings = gb;
   1109 		struct qbe_value item_qv = mklval(ctx, &gb->value);
   1110 		struct qbe_value offs = constl(unpack->offset);
   1111 		pushprei(ctx->current, &item_qv, Q_ADD, &tuple_qv, &offs, NULL);
   1112 	}
   1113 }
   1114 
   1115 static struct gen_value
   1116 gen_expr_binding(struct gen_context *ctx, const struct expression *expr)
   1117 {
   1118 	for (const struct expression_binding *binding = &expr->binding;
   1119 			binding; binding = binding->next) {
   1120 		if (binding->unpack) {
   1121 			if (binding->unpack->object->otype == O_DECL) {
   1122 				gen_expr_binding_unpack_static(ctx, binding);
   1123 			} else {
   1124 				gen_expr_binding_unpack(ctx, binding);
   1125 			}
   1126 			continue;
   1127 		}
   1128 
   1129 		if (binding->object->otype == O_DECL) {
   1130 			// static binding
   1131 			struct declaration decl = {
   1132 				.type = DECL_GLOBAL,
   1133 				.ident = binding->object->ident,
   1134 				.global = {
   1135 					.type = binding->object->type,
   1136 					.value = binding->initializer,
   1137 				},
   1138 			};
   1139 			gen_global_decl(ctx, &decl);
   1140 			continue;
   1141 		}
   1142 
   1143 		const struct type *type = binding->initializer->result;
   1144 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   1145 		gb->value.kind = GV_TEMP;
   1146 		gb->value.type = type;
   1147 		gb->value.name = gen_name(ctx, "binding.%d");
   1148 		gb->object = binding->object;
   1149 		gb->next = ctx->bindings;
   1150 		ctx->bindings = gb;
   1151 
   1152 		struct qbe_value qv = mklval(ctx, &gb->value);
   1153 		struct qbe_value sz = constl(type->size);
   1154 		enum qbe_instr alloc = alloc_for_align(type->align);
   1155 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   1156 		gen_expr_at(ctx, binding->initializer, gb->value);
   1157 	}
   1158 	return gv_void;
   1159 }
   1160 
   1161 static struct gen_value
   1162 gen_expr_control(struct gen_context *ctx, const struct expression *expr)
   1163 {
   1164 	struct gen_scope *scope = gen_scope_lookup(ctx, expr->control.scope);
   1165 
   1166 	if (expr->control.value) {
   1167 		struct gen_value result = gen_expr_with(ctx,
   1168 			expr->control.value, scope->out);
   1169 		branch_copyresult(ctx, result,
   1170 			scope->result, scope->out);
   1171 	}
   1172 
   1173 	struct gen_scope *deferred = ctx->scope;
   1174 	while (deferred != NULL) {
   1175 		gen_defers(ctx, deferred);
   1176 		if (deferred == scope) {
   1177 			break;
   1178 		}
   1179 		deferred = deferred->parent;
   1180 	}
   1181 
   1182 	switch (expr->type) {
   1183 	case EXPR_BREAK:
   1184 		assert(scope->scope->class == SCOPE_LOOP);
   1185 		pushi(ctx->current, NULL, Q_JMP, scope->end, NULL);
   1186 		break;
   1187 	case EXPR_CONTINUE:
   1188 		assert(scope->scope->class == SCOPE_LOOP);
   1189 		pushi(ctx->current, NULL, Q_JMP, scope->after, NULL);
   1190 		break;
   1191 	case EXPR_YIELD:
   1192 		assert(scope->scope->class == SCOPE_COMPOUND);
   1193 		pushi(ctx->current, NULL, Q_JMP, scope->end, NULL);
   1194 		break;
   1195 	default: abort(); // Invariant
   1196 	}
   1197 	return gv_void;
   1198 }
   1199 
   1200 static struct gen_value
   1201 gen_expr_call(struct gen_context *ctx, const struct expression *expr)
   1202 {
   1203 	struct gen_value lvalue = gen_expr(ctx, expr->call.lvalue);
   1204 	lvalue = gen_autoderef(ctx, lvalue);
   1205 
   1206 	const struct type *rtype = type_dealias(lvalue.type);
   1207 	assert(rtype->storage == STORAGE_FUNCTION);
   1208 
   1209 	if (rtype->func.flags & FN_NORETURN) {
   1210 		for (struct gen_scope *scope = ctx->scope; scope;
   1211 				scope = scope->parent) {
   1212 			gen_defers(ctx, scope);
   1213 		}
   1214 	}
   1215 
   1216 	struct qbe_statement call = {
   1217 		.type = Q_INSTR,
   1218 		.instr = Q_CALL,
   1219 	};
   1220 	struct gen_value rval = gv_void;
   1221 	if (type_dealias(rtype->func.result)->storage != STORAGE_VOID) {
   1222 		rval = mkgtemp(ctx, rtype->func.result, "returns.%d");
   1223 		call.out = xcalloc(1, sizeof(struct qbe_value));
   1224 		*call.out = mkqval(ctx, &rval);
   1225 		call.out->type = qtype_lookup(ctx, rtype->func.result, false);
   1226 	}
   1227 
   1228 	bool cvar = false;
   1229 	struct type_func_param *param = rtype->func.params;
   1230 	struct qbe_arguments *args, **next = &call.args;
   1231 	args = *next = xcalloc(1, sizeof(struct qbe_arguments));
   1232 	args->value = mkqval(ctx, &lvalue);
   1233 	next = &args->next;
   1234 	for (struct call_argument *carg = expr->call.args;
   1235 			carg; carg = carg->next) {
   1236 		args = *next = xcalloc(1, sizeof(struct qbe_arguments));
   1237 		struct gen_value arg = gen_expr(ctx, carg->value);
   1238 		args->value = mkqval(ctx, &arg);
   1239 		args->value.type = qtype_lookup(ctx, carg->value->result, false);
   1240 		next = &args->next;
   1241 		if (param) {
   1242 			param = param->next;
   1243 		}
   1244 		if (!param && !cvar && rtype->func.variadism == VARIADISM_C) {
   1245 			cvar = true;
   1246 			args = *next = xcalloc(1, sizeof(struct qbe_arguments));
   1247 			args->value.kind = QV_VARIADIC;
   1248 			next = &args->next;
   1249 		}
   1250 	}
   1251 	push(&ctx->current->body, &call);
   1252 
   1253 	return rval;
   1254 }
   1255 
   1256 static struct gen_value gen_expr_cast(struct gen_context *ctx,
   1257 		const struct expression *expr);
   1258 
   1259 static struct gen_value gen_subset_match_tests(struct gen_context *ctx,
   1260 	struct qbe_value bmatch, struct qbe_value bnext,
   1261 	struct qbe_value tag, const struct type *type);
   1262 
   1263 static struct gen_value gen_nested_match_tests(struct gen_context *ctx,
   1264 		struct gen_value object, struct qbe_value bmatch,
   1265 		struct qbe_value bnext, struct qbe_value tag,
   1266 		const struct type *type);
   1267 
   1268 static struct gen_value
   1269 gen_type_assertion_or_test(struct gen_context *ctx, const struct expression *expr,
   1270 		struct gen_value base)
   1271 {
   1272 	assert(expr->cast.kind == C_TEST || expr->cast.kind == C_ASSERTION);
   1273 	const struct type *want = expr->cast.secondary;
   1274 	struct qbe_value tag = mkqtmp(ctx,
   1275 		qtype_lookup(ctx, &builtin_type_uint, false), ".%d");
   1276 	enum qbe_instr load = load_for_type(ctx, &builtin_type_uint);
   1277 	struct qbe_value qbase = mkqval(ctx, &base);
   1278 	pushi(ctx->current, &tag, load, &qbase, NULL);
   1279 
   1280 	struct qbe_statement failedl, passedl;
   1281 	struct qbe_value bfailed, bpassed = mklabel(ctx, &passedl, "passed.%d");
   1282 	if (expr->cast.kind == C_ASSERTION) {
   1283 		bfailed = mklabel(ctx, &failedl, "failed.%d");
   1284 	} else {
   1285 		bfailed = bpassed;
   1286 	}
   1287 	struct gen_value result = {0};
   1288 	if (tagged_select_subtype(expr->cast.value->result, want, true)) {
   1289 		result = gen_nested_match_tests(ctx, base, bpassed,
   1290 				bfailed, tag, want);
   1291 	} else if (tagged_subset_compat(expr->cast.value->result, want)) {
   1292 		result = gen_subset_match_tests(ctx, bpassed, bfailed, tag,
   1293 				type_dealias(want));
   1294 	} else {
   1295 		abort();
   1296 	}
   1297 
   1298 	if (expr->cast.kind == C_ASSERTION) {
   1299 		push(&ctx->current->body, &failedl);
   1300 		gen_fixed_abort(ctx, expr->loc, ABORT_TYPE_ASSERTION);
   1301 	}
   1302 	push(&ctx->current->body, &passedl);
   1303 	return result;
   1304 }
   1305 
   1306 static void
   1307 gen_expr_cast_slice_at(struct gen_context *ctx,
   1308 	const struct expression *expr, struct gen_value out)
   1309 {
   1310 	const struct type *to = expr->result,
   1311 		*from = type_dealias(expr->cast.value->result);
   1312 	if (from->storage == STORAGE_POINTER) {
   1313 		from = type_dealias(from->pointer.referent);
   1314 	}
   1315 	assert(from->storage == STORAGE_ARRAY);
   1316 	assert(from->array.length != SIZE_UNDEFINED);
   1317 
   1318 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   1319 	struct qbe_value base = mklval(ctx, &out);
   1320 	struct qbe_value sz = constl(to->size);
   1321 	if (from->array.length == 0) {
   1322 		struct qbe_value tmp = constl(0);
   1323 		pushi(ctx->current, NULL, store, &tmp, &base, NULL);
   1324 	} else {
   1325 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1326 		struct qbe_value qvalue = mkqval(ctx, &value);
   1327 		pushi(ctx->current, NULL, store, &qvalue, &base, NULL);
   1328 	}
   1329 	struct qbe_value qptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   1330 	sz = constl(builtin_type_size.size);
   1331 	struct qbe_value ln = constl(from->array.length);
   1332 	pushi(ctx->current, &qptr, Q_ADD, &base, &sz, NULL);
   1333 	pushi(ctx->current, NULL, store, &ln, &qptr, NULL);
   1334 	pushi(ctx->current, &qptr, Q_ADD, &qptr, &sz, NULL);
   1335 	pushi(ctx->current, NULL, store, &ln, &qptr, NULL);
   1336 }
   1337 
   1338 // Returns true if object's storage can be interpreted as want.
   1339 static bool
   1340 tagged_align_compat(const struct type *object, const struct type *want)
   1341 {
   1342 	assert(type_dealias(object)->storage == STORAGE_TAGGED);
   1343 	assert(type_dealias(want)->storage == STORAGE_TAGGED);
   1344 	return object->align == want->align
   1345 		|| want->size == builtin_type_uint.size;
   1346 }
   1347 
   1348 static void
   1349 gen_expr_cast_tagged_at(struct gen_context *ctx,
   1350 	const struct expression *expr, struct gen_value out)
   1351 {
   1352 	assert(expr->type == EXPR_CAST);
   1353 	const struct type *to = expr->result, *from = expr->cast.value->result;
   1354 	const struct type *subtype = tagged_select_subtype(to, from, true);
   1355 
   1356 	if (!subtype && tagged_align_compat(from, to)) {
   1357 		// Case 1: from is a union whose members are a subset of to, and
   1358 		// the alignment matches, so we can just interpret values of
   1359 		// type 'from' as if it were of type 'to'
   1360 		struct gen_value out2 = out;
   1361 		out2.type = from;
   1362 		gen_expr_at(ctx, expr->cast.value, out2);
   1363 		if (expr->cast.kind == C_ASSERTION) {
   1364 			gen_type_assertion_or_test(ctx, expr, out2);
   1365 		}
   1366 	} else if (!subtype) {
   1367 		// Case 2: like case 1, but with an alignment mismatch; more
   1368 		// work is required.
   1369 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1370 		struct qbe_value qval = mkqval(ctx, &value);
   1371 		if (expr->cast.kind == C_ASSERTION) {
   1372 			gen_type_assertion_or_test(ctx, expr, value);
   1373 		}
   1374 		struct qbe_value qout = mkqval(ctx, &out);
   1375 		struct qbe_value tag = mkqtmp(ctx,
   1376 			qtype_lookup(ctx, &builtin_type_uint, false), "tag.%d");
   1377 		enum qbe_instr load = load_for_type(ctx, &builtin_type_uint);
   1378 		enum qbe_instr store = store_for_type(ctx, &builtin_type_uint);
   1379 		pushi(ctx->current, &tag, load, &qval, NULL);
   1380 		pushi(ctx->current, NULL, store, &tag, &qout, NULL);
   1381 		if (to->size == builtin_type_uint.size ||
   1382 				from->size == builtin_type_uint.size) {
   1383 			// No data area to copy
   1384 			return;
   1385 		}
   1386 
   1387 		subtype = tagged_subset_compat(to, from) ? from : to;
   1388 		const struct type *innertype = type_store_tagged_to_union(
   1389 				ctx->store, type_dealias(subtype));
   1390 		struct gen_value iout = mkgtemp(ctx, innertype, ".%d");
   1391 		struct gen_value ival = mkgtemp(ctx, innertype, ".%d");
   1392 		struct qbe_value qiout = mkqval(ctx, &iout);
   1393 		struct qbe_value qival = mkqval(ctx, &ival);
   1394 		struct qbe_value offs = constl(to->align);
   1395 		pushi(ctx->current, &qiout, Q_ADD, &qout, &offs, NULL);
   1396 		offs = constl(from->align);
   1397 		pushi(ctx->current, &qival, Q_ADD, &qval, &offs, NULL);
   1398 		gen_copy_aligned(ctx, iout, ival);
   1399 	} else {
   1400 		// Case 3: from is a member of to
   1401 		assert(subtype == from); // Lowered by check
   1402 		struct qbe_value qout = mkqval(ctx, &out);
   1403 		struct qbe_value id = constw(subtype->id);
   1404 		enum qbe_instr store = store_for_type(ctx, &builtin_type_uint);
   1405 		pushi(ctx->current, NULL, store, &id, &qout, NULL);
   1406 		if (subtype->size == 0) {
   1407 			gen_expr(ctx, expr->cast.value); // side-effects
   1408 			return;
   1409 		}
   1410 
   1411 		struct gen_value storage = mkgtemp(ctx, subtype, ".%d");
   1412 		struct qbe_value qstor = mklval(ctx, &storage);
   1413 		struct qbe_value offs = constl(to->align);
   1414 		pushi(ctx->current, &qstor, Q_ADD, &qout, &offs, NULL);
   1415 		gen_expr_at(ctx, expr->cast.value, storage);
   1416 	}
   1417 }
   1418 
   1419 static bool
   1420 cast_prefers_at(const struct expression *expr)
   1421 {
   1422 	const struct type *to = expr->result, *from = expr->cast.value->result;
   1423 	if (expr->cast.kind == C_TEST) {
   1424 		return false;
   1425 	}
   1426 	// tagged => *; subtype compatible
   1427 	if (type_dealias(from)->storage == STORAGE_TAGGED
   1428 			&& tagged_select_subtype(from, to, true)) {
   1429 		return false;
   1430 	}
   1431 	// * => tagged
   1432 	if (type_dealias(to)->storage == STORAGE_TAGGED) {
   1433 		return true;
   1434 	}
   1435 	// array => array
   1436 	if (type_dealias(to)->storage == STORAGE_ARRAY
   1437 			&& type_dealias(from)->storage == STORAGE_ARRAY) {
   1438 		return true;
   1439 	}
   1440 	// array => slice
   1441 	if (type_dealias(to)->storage == STORAGE_SLICE) {
   1442 		switch (type_dealias(from)->storage) {
   1443 		case STORAGE_ARRAY:
   1444 			return true;
   1445 		case STORAGE_POINTER:
   1446 			from = type_dealias(from)->pointer.referent;
   1447 			return type_dealias(from)->storage == STORAGE_ARRAY;
   1448 		default:
   1449 			return false;
   1450 		}
   1451 	}
   1452 	return false;
   1453 }
   1454 
   1455 static void
   1456 gen_expr_cast_array_at(struct gen_context *ctx,
   1457 	const struct expression *expr, struct gen_value out)
   1458 {
   1459 	const struct type *typeout = type_dealias(expr->result);
   1460 	const struct type *typein = type_dealias(expr->cast.value->result);
   1461 	gen_expr_at(ctx, expr->cast.value, out);
   1462 	if (!typein->array.expandable) {
   1463 		return;
   1464 	}
   1465 
   1466 	assert(typein->array.length != SIZE_UNDEFINED
   1467 			&& typeout->array.length != SIZE_UNDEFINED);
   1468 	assert(typeout->array.length >= typein->array.length);
   1469 
   1470 	const struct type *membtype = typein->array.members;
   1471 	size_t remain = typeout->array.length - typein->array.length;
   1472 
   1473 	struct qbe_value base = mkqval(ctx, &out);
   1474 	struct qbe_value offs = constl((typein->array.length - 1) * membtype->size);
   1475 	struct gen_value next = mkgtemp(ctx, membtype, ".%d");
   1476 	struct qbe_value ptr = mklval(ctx, &next);
   1477 	struct gen_value item = mkgtemp(ctx, membtype, "item.%d");
   1478 	struct qbe_value qitem = mklval(ctx, &item);
   1479 	pushi(ctx->current, &qitem, Q_ADD, &base, &offs, NULL);
   1480 
   1481 	if (remain * membtype->size <= 128) {
   1482 		struct gen_value last = gen_load(ctx, item);
   1483 		for (size_t n = typein->array.length; n < typeout->array.length; ++n) {
   1484 			struct qbe_value offs = constl(n * membtype->size);
   1485 			pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1486 			gen_store(ctx, next, last);
   1487 		}
   1488 		return;
   1489 	}
   1490 
   1491 	offs = constl(typein->array.length * membtype->size);
   1492 	pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1493 
   1494 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memcpy");
   1495 	struct qbe_value dtemp = mklval(ctx, &next);
   1496 	struct qbe_value stemp = mklval(ctx, &item);
   1497 	struct qbe_value sz = constl(remain * membtype->size);
   1498 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &dtemp, &stemp, &sz, NULL);
   1499 }
   1500 
   1501 static void
   1502 gen_expr_cast_at(struct gen_context *ctx,
   1503 	const struct expression *expr, struct gen_value out)
   1504 {
   1505 	if (!cast_prefers_at(expr)) {
   1506 		struct gen_value result = gen_expr_cast(ctx, expr);
   1507 		if (!expr->terminates) {
   1508 			gen_store(ctx, out, result);
   1509 		}
   1510 		return;
   1511 	}
   1512 
   1513 	if (expr->cast.lowered) {
   1514 		pushc(ctx->current, "gen lowered cast");
   1515 	}
   1516 
   1517 	const struct type *to = expr->result;
   1518 	switch (type_dealias(to)->storage) {
   1519 	case STORAGE_SLICE:
   1520 		gen_expr_cast_slice_at(ctx, expr, out);
   1521 		break;
   1522 	case STORAGE_TAGGED:
   1523 		gen_expr_cast_tagged_at(ctx, expr, out);
   1524 		break;
   1525 	case STORAGE_ARRAY:
   1526 		gen_expr_cast_array_at(ctx, expr, out);
   1527 		break;
   1528 	default: abort(); // Invariant
   1529 	}
   1530 }
   1531 
   1532 static struct qbe_value nested_tagged_offset(const struct type *tu,
   1533 		const struct type *targed);
   1534 
   1535 static struct gen_value
   1536 gen_expr_cast(struct gen_context *ctx, const struct expression *expr)
   1537 {
   1538 	const struct type *to = expr->cast.secondary,
   1539 		*from = expr->cast.value->result;
   1540 	if (expr->cast.kind != C_CAST) {
   1541 		bool is_valid_tagged, is_valid_pointer;
   1542 		is_valid_tagged = type_dealias(from)->storage == STORAGE_TAGGED
   1543 				&& (tagged_select_subtype(from, to, true)
   1544 				|| tagged_subset_compat(from, to));
   1545 		is_valid_pointer = type_dealias(from)->storage == STORAGE_POINTER
   1546 				&& (type_dealias(to)->storage == STORAGE_POINTER
   1547 				|| type_dealias(to)->storage == STORAGE_NULL);
   1548 		assert(is_valid_tagged || is_valid_pointer);
   1549 		if (expr->cast.kind == C_TEST && is_valid_tagged) {
   1550 			return gen_type_assertion_or_test(ctx, expr,
   1551 					gen_expr(ctx, expr->cast.value));
   1552 		}
   1553 	}
   1554 
   1555 	if (cast_prefers_at(expr)) {
   1556 		struct gen_value out = mkgtemp(ctx, expr->result, "object.%d");
   1557 		struct qbe_value base = mkqval(ctx, &out);
   1558 		struct qbe_value sz = constl(expr->result->size);
   1559 		enum qbe_instr alloc = alloc_for_align(expr->result->align);
   1560 		pushprei(ctx->current, &base, alloc, &sz, NULL);
   1561 		gen_expr_cast_at(ctx, expr, out);
   1562 		return out;
   1563 	}
   1564 
   1565 	if (expr->cast.lowered) {
   1566 		pushc(ctx->current, "gen lowered cast");
   1567 	}
   1568 
   1569 	// Special cases
   1570 	bool want_null = false;
   1571 	switch (type_dealias(to)->storage) {
   1572 	case STORAGE_NULL:
   1573 		want_null = true;
   1574 		// fallthrough
   1575 	case STORAGE_POINTER:
   1576 		if (type_dealias(from)->storage == STORAGE_SLICE) {
   1577 			struct gen_value value = gen_expr(ctx, expr->cast.value);
   1578 			value.type = to;
   1579 			return gen_load(ctx, value);
   1580 		}
   1581 		if (type_dealias(from)->storage != STORAGE_POINTER) {
   1582 			break;
   1583 		}
   1584 
   1585 		struct gen_value val = gen_expr(ctx, expr->cast.value);
   1586 		struct qbe_value qval = mkqval(ctx, &val);
   1587 		struct qbe_value zero = constl(0);
   1588 		enum qbe_instr compare = want_null ? Q_CEQL : Q_CNEL;
   1589 		if (expr->cast.kind == C_TEST) {
   1590 			struct gen_value out =
   1591 				mkgtemp(ctx, &builtin_type_bool, ".%d");
   1592 			struct qbe_value qout = mkqval(ctx, &out);
   1593 
   1594 			pushi(ctx->current, &qout, compare, &qval, &zero, NULL);
   1595 			return out;
   1596 		} else if (expr->cast.kind == C_ASSERTION) {
   1597 			struct qbe_statement failedl, passedl;
   1598 			struct qbe_value bfailed =
   1599 				mklabel(ctx, &failedl, "failed.%d");
   1600 			struct qbe_value bpassed =
   1601 				mklabel(ctx, &passedl, "passed.%d");
   1602 
   1603 			struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
   1604 			pushi(ctx->current, &cmpres, compare, &qval, &zero, NULL);
   1605 			pushi(ctx->current, NULL, Q_JNZ, &cmpres,
   1606 				&bpassed, &bfailed, NULL);
   1607 			push(&ctx->current->body, &failedl);
   1608 			gen_fixed_abort(ctx, expr->loc, ABORT_TYPE_ASSERTION);
   1609 
   1610 			push(&ctx->current->body, &passedl);
   1611 			if (want_null) {
   1612 				return (struct gen_value){
   1613 					.kind = GV_CONST,
   1614 					.type = &builtin_type_null,
   1615 					.lval = 0,
   1616 				};
   1617 			}
   1618 		}
   1619 		val.type = to;
   1620 		return val;
   1621 	case STORAGE_VOID:
   1622 		gen_expr(ctx, expr->cast.value); // Side-effects
   1623 		return gv_void;
   1624 	default: break;
   1625 	}
   1626 
   1627 	// Special case: tagged => non-tagged
   1628 	if (type_dealias(from)->storage == STORAGE_TAGGED) {
   1629 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1630 		struct qbe_value base = mkcopy(ctx, &value, ".%d");
   1631 		if (expr->cast.kind == C_ASSERTION) {
   1632 			gen_type_assertion_or_test(ctx, expr, value);
   1633 		}
   1634 
   1635 		struct qbe_value align = nested_tagged_offset(
   1636 				expr->cast.value->result, expr->cast.secondary);
   1637 		pushi(ctx->current, &base, Q_ADD, &base, &align, NULL);
   1638 		struct gen_value storage = (struct gen_value){
   1639 			.kind = GV_TEMP,
   1640 			.type = to,
   1641 			.name = base.name,
   1642 		};
   1643 		return gen_load(ctx, storage);
   1644 	}
   1645 
   1646 	// Special case: no conversion required
   1647 	if (type_dealias(to)->storage == type_dealias(from)->storage
   1648 			&& to->size == from->size) {
   1649 		struct gen_value value = gen_expr(ctx, expr->cast.value);
   1650 		value.type = to;
   1651 		return value;
   1652 	}
   1653 
   1654 	struct gen_value value = gen_expr(ctx, expr->cast.value);
   1655 	struct qbe_value qvalue = mkqval(ctx, &value);
   1656 	struct gen_value result = mkgtemp(ctx, expr->result, "cast.%d");
   1657 	struct qbe_value qresult = mkqval(ctx, &result);
   1658 	struct gen_value intermediate;
   1659 	struct qbe_value qintermediate;
   1660 
   1661 	from = lower_const(from, NULL);
   1662 
   1663 	enum qbe_instr op;
   1664 	bool is_signed = type_is_signed(from);
   1665 	enum type_storage fstor = type_dealias(from)->storage,
   1666 		tstor = type_dealias(to)->storage;
   1667 	switch (tstor) {
   1668 	case STORAGE_CHAR:
   1669 	case STORAGE_ENUM:
   1670 	case STORAGE_U8:
   1671 	case STORAGE_I8:
   1672 	case STORAGE_I16:
   1673 	case STORAGE_U16:
   1674 	case STORAGE_I32:
   1675 	case STORAGE_U32:
   1676 	case STORAGE_INT:
   1677 	case STORAGE_UINT:
   1678 	case STORAGE_I64:
   1679 	case STORAGE_U64:
   1680 	case STORAGE_UINTPTR:
   1681 	case STORAGE_RUNE:
   1682 	case STORAGE_SIZE:
   1683 		if (type_is_integer(from) && to->size <= from->size) {
   1684 			op = Q_COPY;
   1685 		} else if (type_is_integer(from) && to->size > from->size) {
   1686 			switch (from->size) {
   1687 			case 4: op = is_signed ? Q_EXTSW : Q_EXTUW; break;
   1688 			case 2: op = is_signed ? Q_EXTSH : Q_EXTUH; break;
   1689 			case 1: op = is_signed ? Q_EXTSB : Q_EXTUB; break;
   1690 			default: abort(); // Invariant
   1691 			}
   1692 		} else if (fstor == STORAGE_POINTER || fstor == STORAGE_NULL) {
   1693 			assert(tstor == STORAGE_UINTPTR);
   1694 			op = Q_COPY;
   1695 		} else if (fstor == STORAGE_RUNE) {
   1696 			op = Q_COPY;
   1697 		} else if (type_is_float(from)) {
   1698 			if (type_is_signed(to)) {
   1699 				switch (fstor) {
   1700 				case STORAGE_F32: op = Q_STOSI; break;
   1701 				case STORAGE_F64: op = Q_DTOSI; break;
   1702 				default: abort(); // Invariant
   1703 				}
   1704 			} else {
   1705 				switch (fstor) {
   1706 				case STORAGE_F32: op = Q_STOUI; break;
   1707 				case STORAGE_F64: op = Q_DTOUI; break;
   1708 				default: abort(); // Invariant
   1709 				}
   1710 			}
   1711 		} else {
   1712 			abort(); // Invariant
   1713 		}
   1714 		pushi(ctx->current, &qresult, op, &qvalue, NULL);
   1715 		break;
   1716 	case STORAGE_F32:
   1717 	case STORAGE_F64:
   1718 		if (type_is_float(from) && from->size == to->size) {
   1719 			op = Q_COPY;
   1720 		} else if (type_is_float(from) && to->size < from->size) {
   1721 			op = Q_TRUNCD;
   1722 		} else if (type_is_float(from) && to->size > from->size) {
   1723 			op = Q_EXTS;
   1724 		} else if (type_is_integer(from)) {
   1725 			if (type_is_signed(from)) {
   1726 				switch (from->size) {
   1727 				case 1:
   1728 				case 2:
   1729 					intermediate = mkgtemp(ctx,
   1730 						&builtin_type_i32, "cast.%d");
   1731 					qintermediate = mkqval(ctx, &intermediate);
   1732 					pushi(ctx->current, &qintermediate,
   1733 						from->size == 1? Q_EXTSB : Q_EXTSH,
   1734 						&qvalue, NULL);
   1735 					qvalue = qintermediate;
   1736 					/* fallthrough */
   1737 				case 4:
   1738 					op = Q_SWTOF;
   1739 					break;
   1740 				case 8:
   1741 					op = Q_SLTOF;
   1742 					break;
   1743 				default: abort(); // Invariant
   1744 				}
   1745 			} else {
   1746 				switch (from->size) {
   1747 				case 1:
   1748 				case 2:
   1749 					intermediate = mkgtemp(ctx,
   1750 						&builtin_type_i32, "cast.%d");
   1751 					qintermediate = mkqval(ctx, &intermediate);
   1752 					pushi(ctx->current, &qintermediate,
   1753 						from->size == 1? Q_EXTUB : Q_EXTUH,
   1754 						&qvalue, NULL);
   1755 					qvalue = qintermediate;
   1756 					/* fallthrough */
   1757 				case 4:
   1758 					op = Q_UWTOF;
   1759 					break;
   1760 				case 8:
   1761 					op = Q_ULTOF;
   1762 					break;
   1763 				default: abort(); // Invariant
   1764 				}
   1765 			}
   1766 		} else {
   1767 			abort(); // Invariant
   1768 		}
   1769 		pushi(ctx->current, &qresult, op, &qvalue, NULL);
   1770 		break;
   1771 	case STORAGE_NULL:
   1772 	case STORAGE_POINTER:
   1773 		pushi(ctx->current, &qresult, Q_COPY, &qvalue, NULL);
   1774 		break;
   1775 	case STORAGE_ARRAY:
   1776 		assert(from->storage == STORAGE_ARRAY);
   1777 		pushi(ctx->current, &qresult, Q_COPY, &qvalue, NULL);
   1778 		break;
   1779 	case STORAGE_SLICE:
   1780 		assert(from->storage == STORAGE_SLICE);
   1781 		pushi(ctx->current, &qresult, Q_COPY, &qvalue, NULL);
   1782 		break;
   1783 	case STORAGE_ALIAS:
   1784 	case STORAGE_BOOL:
   1785 	case STORAGE_ERROR:
   1786 	case STORAGE_FCONST:
   1787 	case STORAGE_FUNCTION:
   1788 	case STORAGE_ICONST:
   1789 	case STORAGE_RCONST:
   1790 	case STORAGE_STRING:
   1791 	case STORAGE_STRUCT:
   1792 	case STORAGE_TAGGED:
   1793 	case STORAGE_TUPLE:
   1794 	case STORAGE_UNION:
   1795 	case STORAGE_VALIST:
   1796 	case STORAGE_VOID:
   1797 		abort(); // Invariant
   1798 	}
   1799 
   1800 	return result;
   1801 }
   1802 
   1803 static struct gen_value
   1804 gen_expr_compound_with(struct gen_context *ctx,
   1805 	const struct expression *expr,
   1806 	struct gen_value *out)
   1807 {
   1808 	struct qbe_statement lend;
   1809 	struct qbe_value bend = mklabel(ctx, &lend, ".%d");
   1810 	struct gen_scope *scope = push_scope(ctx, expr->compound.scope);
   1811 	scope->end = &bend;
   1812 
   1813 	struct gen_value gvout = gv_void;
   1814 	if (!out) {
   1815 		gvout = mkgtemp(ctx, expr->result, ".%d");
   1816 	}
   1817 	scope->out = out;
   1818 	scope->result = gvout;
   1819 
   1820 	const struct expressions *exprs;
   1821 	for (exprs = &expr->compound.exprs; exprs->next; exprs = exprs->next) {
   1822 		gen_expr(ctx, exprs->expr);
   1823 	}
   1824 
   1825 	struct gen_value last = gen_expr_with(ctx, exprs->expr, out);
   1826 	branch_copyresult(ctx, last, gvout, out);
   1827 	pop_scope(ctx, !exprs->expr->terminates);
   1828 	push(&ctx->current->body, &lend);
   1829 	return gvout;
   1830 }
   1831 
   1832 static void
   1833 gen_const_array_at(struct gen_context *ctx,
   1834 	const struct expression *expr,
   1835 	struct gen_value out)
   1836 {
   1837 	struct array_constant *aexpr = expr->constant.array;
   1838 	struct qbe_value base = mkqval(ctx, &out);
   1839 
   1840 	size_t n = 0;
   1841 	const struct type *atype = type_dealias(expr->result);
   1842 	size_t msize = atype->array.members->size;
   1843 	struct gen_value item = mkgtemp(ctx, atype->array.members, "item.%d");
   1844 	for (const struct array_constant *ac = aexpr; ac; ac = ac->next) {
   1845 		struct qbe_value offs = constl(n * msize);
   1846 		struct qbe_value ptr = mklval(ctx, &item);
   1847 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1848 		gen_expr_at(ctx, ac->value, item);
   1849 		++n;
   1850 	}
   1851 	assert(n == atype->array.length);
   1852 	if (!atype->array.expandable || n == 0) {
   1853 		return;
   1854 	}
   1855 	assert(out.type);
   1856 	const struct type_array arr = type_dealias(out.type)->array;
   1857 	if (arr.length <= n) {
   1858 		return;
   1859 	}
   1860 
   1861 	// last copied element
   1862 	struct qbe_value lsize = constl((n - 1) * msize);
   1863 	struct qbe_value last = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   1864 	pushi(ctx->current, &last, Q_ADD, &base, &lsize, NULL);
   1865 	// start from the elements already copied
   1866 	struct qbe_value nsize = constl(n * msize);
   1867 	struct qbe_value next = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   1868 	pushi(ctx->current, &next, Q_ADD, &base, &nsize, NULL);
   1869 
   1870 	struct qbe_value rtmemcpy = mkrtfunc(ctx, "rt.memcpy");
   1871 	struct qbe_value qlen = constl((arr.length - n) * msize);
   1872 	pushi(ctx->current, NULL, Q_CALL, &rtmemcpy, &next, &last, &qlen, NULL);
   1873 }
   1874 
   1875 static void
   1876 gen_const_string_at(struct gen_context *ctx,
   1877 	const struct expression *expr, struct gen_value out)
   1878 {
   1879 	const struct expression_constant *constexpr = &expr->constant;
   1880 	const char *val = constexpr->string.value;
   1881 	size_t len = constexpr->string.len;
   1882 
   1883 	// TODO: Generate string data structure as global also?
   1884 	struct qbe_value global = mkqtmp(ctx, ctx->arch.ptr, "strdata.%d");
   1885 	global.kind = QV_GLOBAL;
   1886 
   1887 	struct qbe_def *def = xcalloc(1, sizeof(struct qbe_def));
   1888 	def->name = global.name;
   1889 	def->kind = Q_DATA;
   1890 	def->data.align = ALIGN_UNDEFINED;
   1891 	def->data.items.type = QD_STRING;
   1892 	def->data.items.str = xcalloc(1, len);
   1893 	memcpy(def->data.items.str, val, len);
   1894 	def->data.items.sz = len;
   1895 
   1896 	if (len != 0) {
   1897 		qbe_append_def(ctx->out, def);
   1898 	} else {
   1899 		free(def);
   1900 		global = constl(0);
   1901 	}
   1902 
   1903 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   1904 	struct qbe_value strp = mkcopy(ctx, &out, ".%d");
   1905 	struct qbe_value qlen = constl(len);
   1906 	struct qbe_value offs = constl(builtin_type_size.size);
   1907 	pushi(ctx->current, NULL, store, &global, &strp, NULL);
   1908 	pushi(ctx->current, &strp, Q_ADD, &strp, &offs, NULL);
   1909 	pushi(ctx->current, NULL, store, &qlen, &strp, NULL);
   1910 	pushi(ctx->current, &strp, Q_ADD, &strp, &offs, NULL);
   1911 	pushi(ctx->current, NULL, store, &qlen, &strp, NULL);
   1912 }
   1913 
   1914 static void
   1915 gen_const_struct_at(struct gen_context *ctx,
   1916 	const struct expression *expr, struct gen_value out)
   1917 {
   1918 	// TODO: Merge me into constant expressions
   1919 	struct qbe_value base = mkqval(ctx, &out);
   1920 
   1921 	struct gen_value ftemp = mkgtemp(ctx, &builtin_type_void, "field.%d");
   1922 	for (const struct struct_constant *field = expr->constant._struct;
   1923 			field; field = field->next) {
   1924 		assert(field->value);
   1925 
   1926 		struct qbe_value offs = constl(field->field->offset);
   1927 		ftemp.type = field->value->result;
   1928 		struct qbe_value ptr = mklval(ctx, &ftemp);
   1929 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1930 		gen_expr_at(ctx, field->value, ftemp);
   1931 	}
   1932 }
   1933 
   1934 static void
   1935 gen_const_tagged_at(struct gen_context *ctx,
   1936 	const struct expression *expr, struct gen_value out)
   1937 {
   1938 	struct qbe_value qout = mklval(ctx, &out);
   1939 	const struct type *subtype = expr->constant.tagged.tag;
   1940 	struct qbe_value id = constw(subtype->id);
   1941 	enum qbe_instr store = store_for_type(ctx, &builtin_type_uint);
   1942 	pushi(ctx->current, NULL, store, &id, &qout, NULL);
   1943 	if (subtype->size == 0) {
   1944 		return;
   1945 	}
   1946 
   1947 	struct gen_value storage = mkgtemp(ctx, subtype, ".%d");
   1948 	struct qbe_value qstor = mklval(ctx, &storage);
   1949 	struct qbe_value offs = constl(expr->result->align);
   1950 	pushi(ctx->current, &qstor, Q_ADD, &qout, &offs, NULL);
   1951 	gen_expr_at(ctx, expr->constant.tagged.value, storage);
   1952 }
   1953 
   1954 static void
   1955 gen_const_tuple_at(struct gen_context *ctx,
   1956 	const struct expression *expr, struct gen_value out)
   1957 {
   1958 	// TODO: Merge me into constant expressions
   1959 	struct qbe_value base = mkqval(ctx, &out);
   1960 
   1961 	struct gen_value ftemp = mkgtemp(ctx, &builtin_type_void, "field.%d");
   1962 	for (const struct tuple_constant *field = expr->constant.tuple; field;
   1963 			field = field->next) {
   1964 		assert(field->value);
   1965 
   1966 		struct qbe_value offs = constl(field->field->offset);
   1967 		ftemp.type = field->value->result;
   1968 		struct qbe_value ptr = mklval(ctx, &ftemp);
   1969 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   1970 		gen_expr_at(ctx, field->value, ftemp);
   1971 	}
   1972 }
   1973 
   1974 static void
   1975 gen_expr_const_at(struct gen_context *ctx,
   1976 	const struct expression *expr, struct gen_value out)
   1977 {
   1978 	if (!type_is_aggregate(type_dealias(expr->result))) {
   1979 		gen_store(ctx, out, gen_expr(ctx, expr));
   1980 		return;
   1981 	}
   1982 
   1983 	switch (type_dealias(expr->result)->storage) {
   1984 	case STORAGE_ARRAY:
   1985 		gen_const_array_at(ctx, expr, out);
   1986 		break;
   1987 	case STORAGE_STRING:
   1988 		gen_const_string_at(ctx, expr, out);
   1989 		break;
   1990 	case STORAGE_STRUCT:
   1991 		gen_const_struct_at(ctx, expr, out);
   1992 		break;
   1993 	case STORAGE_TAGGED:
   1994 		gen_const_tagged_at(ctx, expr, out);
   1995 		break;
   1996 	case STORAGE_TUPLE:
   1997 		gen_const_tuple_at(ctx, expr, out);
   1998 		break;
   1999 	default:
   2000 		abort(); // Invariant
   2001 	}
   2002 }
   2003 
   2004 static struct gen_value
   2005 gen_expr_const(struct gen_context *ctx, const struct expression *expr)
   2006 {
   2007 	if (type_is_aggregate(type_dealias(expr->result))) {
   2008 		struct gen_value out = mkgtemp(ctx, expr->result, "object.%d");
   2009 		struct qbe_value base = mkqval(ctx, &out);
   2010 		struct qbe_value sz = constl(expr->result->size);
   2011 		enum qbe_instr alloc = alloc_for_align(expr->result->align);
   2012 		pushprei(ctx->current, &base, alloc, &sz, NULL);
   2013 		gen_expr_at(ctx, expr, out);
   2014 		return out;
   2015 	}
   2016 
   2017 	struct gen_value val = {
   2018 		.kind = GV_CONST,
   2019 		.type = expr->result,
   2020 	};
   2021 
   2022 	// Special cases
   2023 	switch (type_dealias(expr->result)->storage) {
   2024 	case STORAGE_BOOL:
   2025 		val.wval = expr->constant.bval ? 1 : 0;
   2026 		return val;
   2027 	case STORAGE_VOID:
   2028 		return val;
   2029 	case STORAGE_NULL:
   2030 		val.lval = 0;
   2031 		return val;
   2032 	default:
   2033 		// Moving right along
   2034 		break;
   2035 	}
   2036 
   2037 	if (expr->constant.object != NULL) {
   2038 		assert(expr->constant.ival == 0);
   2039 		val = gen_access_ident(ctx, expr->constant.object);
   2040 		val.type = expr->result;
   2041 		return val;
   2042 	}
   2043 
   2044 	const struct qbe_type *qtype = qtype_lookup(ctx, expr->result, false);
   2045 	switch (qtype->stype) {
   2046 	case Q_BYTE:
   2047 	case Q_HALF:
   2048 	case Q_WORD:
   2049 		val.wval = (uint32_t)expr->constant.uval;
   2050 		return val;
   2051 	case Q_LONG:
   2052 		val.lval = expr->constant.uval;
   2053 		return val;
   2054 	case Q_SINGLE:
   2055 		val.sval = (float)expr->constant.fval;
   2056 		return val;
   2057 	case Q_DOUBLE:
   2058 		val.dval = expr->constant.fval;
   2059 		return val;
   2060 	case Q__VOID:
   2061 		return val;
   2062 	case Q__AGGREGATE:
   2063 		assert(0); // Invariant
   2064 	}
   2065 
   2066 	abort(); // Invariant
   2067 }
   2068 
   2069 static struct gen_value
   2070 gen_expr_defer(struct gen_context *ctx, const struct expression *expr)
   2071 {
   2072 	struct gen_defer *defer = xcalloc(1, sizeof(struct gen_defer));
   2073 	defer->expr = expr->defer.deferred;
   2074 	defer->next = ctx->scope->defers;
   2075 	ctx->scope->defers = defer;
   2076 	return gv_void;
   2077 }
   2078 
   2079 static struct gen_value
   2080 gen_expr_delete(struct gen_context *ctx, const struct expression *expr)
   2081 {
   2082 	struct gen_value object, start;
   2083 	const struct expression *dexpr = expr->delete.expr;
   2084 	if (dexpr->type == EXPR_SLICE) {
   2085 		object = gen_expr(ctx, dexpr->slice.object);
   2086 		if (dexpr->slice.start) {
   2087 			start = gen_expr(ctx, dexpr->slice.start);
   2088 		} else {
   2089 			start.kind = GV_CONST;
   2090 			start.type = &builtin_type_size;
   2091 			start.lval = 0;
   2092 		}
   2093 	} else {
   2094 		assert(dexpr->type == EXPR_ACCESS
   2095 			&& dexpr->access.type == ACCESS_INDEX);
   2096 		object = gen_expr(ctx, dexpr->access.array);
   2097 		start = gen_expr(ctx, dexpr->access.index);
   2098 	}
   2099 	object = gen_autoderef(ctx, object);
   2100 	assert(type_dealias(object.type)->storage == STORAGE_SLICE);
   2101 
   2102 	struct qbe_value qobj = mkqval(ctx, &object);
   2103 	struct qbe_value qlenptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2104 	struct qbe_value qlen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2105 	struct qbe_value offset = constl(builtin_type_size.size);
   2106 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
   2107 	pushi(ctx->current, &qlenptr, Q_ADD, &qobj, &offset, NULL);
   2108 	pushi(ctx->current, &qlen, load, &qlenptr, NULL);
   2109 
   2110 	struct qbe_value qstart = mkqval(ctx, &start);
   2111 	struct qbe_value qend = {0};
   2112 	if (dexpr->type == EXPR_SLICE) {
   2113 		if (dexpr->slice.end) {
   2114 			struct gen_value end = gen_expr(ctx, dexpr->slice.end);
   2115 			qend = mkqval(ctx, &end);
   2116 		} else {
   2117 			qend = qlen;
   2118 		}
   2119 	} else {
   2120 		struct qbe_value tmp = constl(1);
   2121 		qend = mkqtmp(ctx, qstart.type, ".%d");
   2122 		pushi(ctx->current, &qend, Q_ADD, &qstart, &tmp, NULL);
   2123 	}
   2124 
   2125 	struct qbe_value start_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2126 	struct qbe_value end_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2127 	struct qbe_value startend_oob = mkqtmp(ctx, &qbe_word, ".%d");
   2128 	struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
   2129 	pushi(ctx->current, &start_oob, Q_CULEL, &qstart, &qlen, NULL);
   2130 	pushi(ctx->current, &end_oob, Q_CULEL, &qend, &qlen, NULL);
   2131 	pushi(ctx->current, &valid, Q_AND, &start_oob, &end_oob, NULL);
   2132 	pushi(ctx->current, &startend_oob, Q_CULEL, &qstart, &qend, NULL);
   2133 	pushi(ctx->current, &valid, Q_AND, &valid, &startend_oob, NULL);
   2134 
   2135 	struct qbe_statement linvalid, lvalid;
   2136 	struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
   2137 	struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
   2138 
   2139 	pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
   2140 	push(&ctx->current->body, &linvalid);
   2141 	gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
   2142 	push(&ctx->current->body, &lvalid);
   2143 
   2144 	struct qbe_value data = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2145 	struct qbe_value startptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2146 	struct qbe_value endptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2147 	struct qbe_value mlen = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2148 	pushi(ctx->current, &data, load, &qobj, NULL);
   2149 	struct qbe_value membsz =
   2150 		constl(type_dealias(object.type)->array.members->size);
   2151 	pushi(ctx->current, &startptr, Q_MUL, &qstart, &membsz, NULL);
   2152 	pushi(ctx->current, &startptr, Q_ADD, &startptr, &data, NULL);
   2153 	pushi(ctx->current, &endptr, Q_MUL, &qend, &membsz, NULL);
   2154 	pushi(ctx->current, &endptr, Q_ADD, &endptr, &data, NULL);
   2155 	pushi(ctx->current, &qlen, Q_SUB, &qlen, &qend, NULL);
   2156 	pushi(ctx->current, &mlen, Q_MUL, &qlen, &membsz, NULL);
   2157 
   2158 	struct qbe_value rtmemmove = mkrtfunc(ctx, "rt.memmove");
   2159 	pushi(ctx->current, NULL, Q_CALL, &rtmemmove, &startptr, &endptr, &mlen,
   2160 		NULL);
   2161 
   2162 	pushi(ctx->current, &qlen, Q_ADD, &qlen, &qstart, NULL);
   2163 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   2164 	pushi(ctx->current, NULL, store, &qlen, &qlenptr, NULL);
   2165 
   2166 	if (!expr->delete.is_static) {
   2167 		struct qbe_value rtunensure = mkrtfunc(ctx, "rt.unensure");
   2168 		qobj = mklval(ctx, &object);
   2169 		pushi(ctx->current, NULL, Q_CALL, &rtunensure, &qobj, &membsz,
   2170 			NULL);
   2171 	}
   2172 
   2173 	return gv_void;
   2174 }
   2175 
   2176 static struct gen_value
   2177 gen_expr_for(struct gen_context *ctx, const struct expression *expr)
   2178 {
   2179 	struct qbe_statement lloop, lbody, lafter, lend;
   2180 	struct qbe_value bloop = mklabel(ctx, &lloop, "loop.%d");
   2181 	struct qbe_value bbody = mklabel(ctx, &lbody, "body.%d");
   2182 	struct qbe_value bend = mklabel(ctx, &lend, ".%d");
   2183 	struct qbe_value bafter = mklabel(ctx, &lafter, "after.%d");
   2184 
   2185 	if (expr->_for.bindings) {
   2186 		gen_expr_binding(ctx, expr->_for.bindings);
   2187 	}
   2188 
   2189 	push_scope(ctx, expr->_for.scope);
   2190 	ctx->scope->after = &bafter;
   2191 	ctx->scope->end = &bend;
   2192 
   2193 	push(&ctx->current->body, &lloop);
   2194 	struct gen_value cond = gen_expr(ctx, expr->_for.cond);
   2195 	struct qbe_value qcond = mkqval(ctx, &cond);
   2196 	pushi(ctx->current, NULL, Q_JNZ, &qcond, &bbody, &bend, NULL);
   2197 
   2198 	push(&ctx->current->body, &lbody);
   2199 	gen_expr(ctx, expr->_for.body);
   2200 
   2201 	push(&ctx->current->body, &lafter);
   2202 	if (expr->_for.afterthought) {
   2203 		gen_expr(ctx, expr->_for.afterthought);
   2204 	}
   2205 
   2206 	pop_scope(ctx, true);
   2207 
   2208 	pushi(ctx->current, NULL, Q_JMP, &bloop, NULL);
   2209 
   2210 	push(&ctx->current->body, &lend);
   2211 	return gv_void;
   2212 }
   2213 
   2214 static struct gen_value
   2215 gen_expr_free(struct gen_context *ctx, const struct expression *expr)
   2216 {
   2217 	const struct type *type = type_dealias(expr->free.expr->result);
   2218 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.free");
   2219 	struct gen_value val = gen_expr(ctx, expr->free.expr);
   2220 	struct qbe_value qval = mkqval(ctx, &val);
   2221 	if (type->storage == STORAGE_SLICE || type->storage == STORAGE_STRING) {
   2222 		struct qbe_value lval = mklval(ctx, &val);
   2223 		qval = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2224 		pushi(ctx->current, &qval, Q_LOADL, &lval, NULL);
   2225 	}
   2226 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &qval, NULL);
   2227 	return gv_void;
   2228 }
   2229 
   2230 static struct gen_value
   2231 gen_expr_if_with(struct gen_context *ctx,
   2232 	const struct expression *expr,
   2233 	struct gen_value *out)
   2234 {
   2235 	struct gen_value gvout = gv_void;
   2236 	if (!out) {
   2237 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2238 	}
   2239 
   2240 	struct qbe_statement ltrue, lfalse, lend;
   2241 	struct qbe_value btrue = mklabel(ctx, &ltrue, "true.%d");
   2242 	struct qbe_value bfalse = mklabel(ctx, &lfalse, "false.%d");
   2243 	struct qbe_value bend = mklabel(ctx, &lend, ".%d");
   2244 	struct gen_value cond = gen_expr(ctx, expr->_if.cond);
   2245 	struct qbe_value qcond = mkqval(ctx, &cond);
   2246 	pushi(ctx->current, NULL, Q_JNZ, &qcond, &btrue, &bfalse, NULL);
   2247 
   2248 	push(&ctx->current->body, &ltrue);
   2249 	struct gen_value vtrue = gen_expr_with(ctx, expr->_if.true_branch, out);
   2250 	branch_copyresult(ctx, vtrue, gvout, out);
   2251 	if (!expr->_if.true_branch->terminates) {
   2252 		pushi(ctx->current, NULL, Q_JMP, &bend, NULL);
   2253 	}
   2254 
   2255 	push(&ctx->current->body, &lfalse);
   2256 	if (expr->_if.false_branch) {
   2257 		struct gen_value vfalse = gen_expr_with(ctx,
   2258 				expr->_if.false_branch, out);
   2259 		branch_copyresult(ctx, vfalse, gvout, out);
   2260 	}
   2261 
   2262 	push(&ctx->current->body, &lend);
   2263 	return gvout;
   2264 }
   2265 
   2266 static struct gen_value
   2267 gen_expr_insert(struct gen_context *ctx, const struct expression *expr)
   2268 {
   2269 	// XXX: A lot of this code is identical to the corresponding append
   2270 	// code, maybe we can/should deduplicate it
   2271 	assert(expr->append.length == NULL);
   2272 	const struct expression *objexpr = expr->append.object;
   2273 	assert(objexpr->type == EXPR_ACCESS
   2274 			&& objexpr->access.type == ACCESS_INDEX);
   2275 	const struct expression *arexpr = objexpr->access.array;
   2276 	struct gen_value slice = gen_expr(ctx, arexpr);
   2277 	slice = gen_autoderef(ctx, slice);
   2278 	struct qbe_value qslice = mkqval(ctx, &slice);
   2279 	struct gen_value index = gen_expr(ctx, objexpr->access.index);
   2280 	struct qbe_value qindex = mkqval(ctx, &index);
   2281 
   2282 	struct qbe_value lenptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2283 	struct qbe_value prevlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2284 	struct qbe_value offs = constl(builtin_type_size.size);
   2285 	pushi(ctx->current, &lenptr, Q_ADD, &qslice, &offs, NULL);
   2286 	enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
   2287 	pushi(ctx->current, &prevlen, load, &lenptr, NULL);
   2288 
   2289 	struct qbe_value appendlen = constl(1);
   2290 	const struct type *valtype = type_dealias(expr->append.value->result);
   2291 
   2292 	struct gen_value value;
   2293 	struct qbe_value qvalue;
   2294 	if (!expr->append.is_multi || valtype->storage != STORAGE_ARRAY) {
   2295 		// We use gen_expr_at for the array case to avoid a copy
   2296 		value = gen_expr(ctx, expr->append.value);
   2297 		qvalue = mkqval(ctx, &value);
   2298 	}
   2299 
   2300 	if (expr->append.is_multi) {
   2301 		if (valtype->storage == STORAGE_ARRAY) {
   2302 			assert(valtype->array.length != SIZE_UNDEFINED);
   2303 			appendlen = constl(valtype->array.length);
   2304 		} else {
   2305 			appendlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2306 			struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2307 			offs = constl(builtin_type_size.size);
   2308 			pushi(ctx->current, &ptr, Q_ADD, &qvalue, &offs, NULL);
   2309 			pushi(ctx->current, &appendlen, load, &ptr, NULL);
   2310 		}
   2311 	}
   2312 
   2313 	struct qbe_value newlen = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2314 	pushi(ctx->current, &newlen, Q_ADD, &prevlen, &appendlen, NULL);
   2315 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   2316 	pushi(ctx->current, NULL, store, &newlen, &lenptr, NULL);
   2317 
   2318 	struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2319 	const struct type *mtype = type_dealias(slice.type)->array.members;
   2320 	struct qbe_value membsz = constl(mtype->size);
   2321 	if (!expr->append.is_static) {
   2322 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.ensure");
   2323 		struct qbe_value lval = mklval(ctx, &slice);
   2324 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &lval, &membsz, NULL);
   2325 	} else {
   2326 		offs = constl(builtin_type_size.size * 2);
   2327 		pushi(ctx->current, &ptr, Q_ADD, &qslice, &offs, NULL);
   2328 		struct qbe_value cap = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2329 		pushi(ctx->current, &cap, load, &ptr, NULL);
   2330 
   2331 		struct qbe_statement lvalid, linvalid;
   2332 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
   2333 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
   2334 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
   2335 		pushi(ctx->current, &valid, Q_CULEL, &newlen, &cap, NULL);
   2336 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
   2337 
   2338 		push(&ctx->current->body, &linvalid);
   2339 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
   2340 		push(&ctx->current->body, &lvalid);
   2341 	}
   2342 
   2343 	struct qbe_value base = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2344 	pushi(ctx->current, &base, load, &qslice, NULL);
   2345 
   2346 	pushi(ctx->current, &ptr, Q_MUL, &qindex, &membsz, NULL);
   2347 	pushi(ctx->current, &ptr, Q_ADD, &base, &ptr, NULL);
   2348 
   2349 	struct qbe_value nbyte = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2350 	struct qbe_value dest = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2351 	struct qbe_value ncopy = mkqtmp(ctx, ctx->arch.sz, ".%d");
   2352 	pushi(ctx->current, &nbyte, Q_MUL, &appendlen, &membsz, NULL);
   2353 	pushi(ctx->current, &ncopy, Q_SUB, &prevlen, &qindex, NULL);
   2354 	pushi(ctx->current, &ncopy, Q_MUL, &ncopy, &membsz, NULL);
   2355 	pushi(ctx->current, &dest, Q_ADD, &ptr, &nbyte, NULL);
   2356 
   2357 	struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memmove");
   2358 	pushi(ctx->current, NULL, Q_CALL, &rtfunc, &dest, &ptr, &ncopy, NULL);
   2359 
   2360 	struct gen_value item = {
   2361 		.kind = GV_TEMP,
   2362 		.type = mtype,
   2363 		.name = ptr.name,
   2364 	};
   2365 	if (expr->append.is_multi && valtype->storage == STORAGE_ARRAY) {
   2366 		item.type = valtype;
   2367 		gen_expr_at(ctx, expr->append.value, item);
   2368 	} else if (expr->append.is_multi && valtype->storage == STORAGE_SLICE) {
   2369 		struct qbe_value qsrc = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2370 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memmove");
   2371 		pushi(ctx->current, &qsrc, load, &qvalue, NULL);
   2372 		pushi(ctx->current, NULL, Q_CALL, &rtfunc, &ptr, &qsrc, &nbyte, NULL);
   2373 	} else {
   2374 		gen_store(ctx, item, value);
   2375 	}
   2376 
   2377 	return gv_void;
   2378 }
   2379 
   2380 enum match_compat {
   2381 	// The case type is a member of the match object type and can be used
   2382 	// directly from the match object's tagged union storage area.
   2383 	COMPAT_SUBTYPE,
   2384 	// The case type is a tagged union which is a subset of the object type,
   2385 	// and alignment-compatible. In this case we can use the match object
   2386 	// directly.
   2387 	COMPAT_ALIGNED,
   2388 	// The case type is a tagged union which is a subset of the object type,
   2389 	// but with a different alignment. In this case we must convert the
   2390 	// match object before using it.
   2391 	COMPAT_MISALIGNED,
   2392 };
   2393 
   2394 static struct qbe_value
   2395 nested_tagged_offset(const struct type *tu, const struct type *target)
   2396 {
   2397 	// This function calculates the offset of a member in a nested tagged union
   2398 	//
   2399 	// type foo = (int | void);
   2400 	// type bar = (size | foo);
   2401 	//
   2402 	// Offset of the foo field from the start of bar is 8 and offset of int
   2403 	// inside foo is 4, so the offset of the int from the start of bar is 12
   2404 	const struct type *test = tu;
   2405 	struct qbe_value offset = constl(tu->align);
   2406 	do {
   2407 		test = tagged_select_subtype(tu, target, false);
   2408 		if (!test) {
   2409 			break;
   2410 		}
   2411 		if (test->id != target->id && type_dealias(test)->id != target->id) {
   2412 			offset.lval += test->align;
   2413 		}
   2414 		tu = test;
   2415 	} while (test->id != target->id && type_dealias(test)->id != target->id);
   2416 	return offset;
   2417 }
   2418 
   2419 static struct gen_value
   2420 gen_nested_match_tests(struct gen_context *ctx, struct gen_value object,
   2421 	struct qbe_value bmatch, struct qbe_value bnext,
   2422 	struct qbe_value tag, const struct type *type)
   2423 {
   2424 	// This function handles the case where we're matching against a type
   2425 	// which is a member of the tagged union, or an inner tagged union.
   2426 	//
   2427 	// type foo = (int | void);
   2428 	// type bar = (size | foo);
   2429 	//
   2430 	// let x: bar = 10i;
   2431 	// match (x) {
   2432 	// case let z: size => ...
   2433 	// case let i: int => ...
   2434 	// case	void => ...
   2435 	// };
   2436 	//
   2437 	// In the first case, we can simply test the object's tag. In the second
   2438 	// case, we have to test if the selected tag is 'foo', then check the
   2439 	// tag of the foo object for int.
   2440 	struct qbe_value *subtag = &tag;
   2441 	struct qbe_value subval = mkcopy(ctx, &object, "subval.%d");
   2442 	struct gen_value match = mkgtemp(ctx, &builtin_type_bool, ".%d");
   2443 	struct qbe_value qmatch = mkqval(ctx, &match);
   2444 	struct qbe_value temp = mkqtmp(ctx, &qbe_word, ".%d");
   2445 	const struct type *subtype = object.type;
   2446 	const struct type *test = type;
   2447 	do {
   2448 		struct qbe_statement lsubtype;
   2449 		struct qbe_value bsubtype = mklabel(ctx, &lsubtype, "subtype.%d");
   2450 
   2451 		if (type_dealias(subtype)->storage != STORAGE_TAGGED) {
   2452 			break;
   2453 		}
   2454 		test = tagged_select_subtype(subtype, type, false);
   2455 		if (!test) {
   2456 			break;
   2457 		}
   2458 
   2459 		struct qbe_value id = constw(test->id);
   2460 		pushi(ctx->current, &qmatch, Q_CEQW, subtag, &id, NULL);
   2461 		pushi(ctx->current, NULL, Q_JNZ, &qmatch, &bsubtype, &bnext, NULL);
   2462 		push(&ctx->current->body, &lsubtype);
   2463 
   2464 		if (test->id != type->id && type_dealias(test)->id != type->id) {
   2465 			struct qbe_value offs = constl(subtype->align);
   2466 			pushi(ctx->current, &subval, Q_ADD, &subval, &offs, NULL);
   2467 			pushi(ctx->current, &temp, Q_LOADUW, &subval, NULL);
   2468 			subtag = &temp;
   2469 		}
   2470 
   2471 		subtype = test;
   2472 	} while (test->id != type->id && type_dealias(test)->id != type->id);
   2473 
   2474 	pushi(ctx->current, NULL, Q_JMP, &bmatch, NULL);
   2475 	return match;
   2476 }
   2477 
   2478 static struct gen_value
   2479 gen_subset_match_tests(struct gen_context *ctx,
   2480 	struct qbe_value bmatch, struct qbe_value bnext,
   2481 	struct qbe_value tag, const struct type *type)
   2482 {
   2483 	// In this case, we're testing a case which is itself a tagged union,
   2484 	// and is a subset of the match object.
   2485 	//
   2486 	// type foo = (size | int | void);
   2487 	//
   2488 	// let x: foo = 10i;
   2489 	// match (x) {
   2490 	// case let n: (size | int) => ...
   2491 	// case void => ...
   2492 	// };
   2493 	//
   2494 	// In this situation, we test the match object's tag against each type
   2495 	// ID of the case type.
   2496 	struct gen_value match = mkgtemp(ctx, &builtin_type_bool, ".%d");
   2497 	for (const struct type_tagged_union *tu = &type->tagged; tu; tu = tu->next) {
   2498 		struct qbe_statement lnexttag;
   2499 		struct qbe_value bnexttag = mklabel(ctx, &lnexttag, ".%d");
   2500 		struct qbe_value id = constl(tu->type->id);
   2501 		struct qbe_value qmatch = mkqval(ctx, &match);
   2502 		pushi(ctx->current, &qmatch, Q_CEQW, &tag, &id, NULL);
   2503 		pushi(ctx->current, NULL, Q_JNZ, &qmatch, &bmatch, &bnexttag, NULL);
   2504 		push(&ctx->current->body, &lnexttag);
   2505 	}
   2506 	pushi(ctx->current, NULL, Q_JMP, &bnext, NULL);
   2507 	return match;
   2508 }
   2509 
   2510 static struct gen_value
   2511 gen_match_with_tagged(struct gen_context *ctx,
   2512 	const struct expression *expr,
   2513 	struct gen_value *out)
   2514 {
   2515 	struct gen_value gvout = gv_void;
   2516 	if (!out) {
   2517 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2518 	}
   2519 
   2520 	const struct type *objtype = expr->match.value->result;
   2521 	struct gen_value object = gen_expr(ctx, expr->match.value);
   2522 	struct qbe_value qobject = mkqval(ctx, &object);
   2523 	struct qbe_value tag = mkqtmp(ctx, ctx->arch.sz, "tag.%d");
   2524 	enum qbe_instr load = load_for_type(ctx, &builtin_type_uint);
   2525 	pushi(ctx->current, &tag, load, &qobject, NULL);
   2526 
   2527 	struct qbe_statement lout;
   2528 	struct qbe_value bout = mklabel(ctx, &lout, ".%d");
   2529 
   2530 	struct gen_value bval;
   2531 	const struct match_case *_default = NULL;
   2532 	for (const struct match_case *_case = expr->match.cases;
   2533 			_case; _case = _case->next) {
   2534 		if (!_case->type) {
   2535 			_default = _case;
   2536 			continue;
   2537 		}
   2538 
   2539 		struct qbe_statement lmatch, lnext;
   2540 		struct qbe_value bmatch = mklabel(ctx, &lmatch, "matches.%d");
   2541 		struct qbe_value bnext = mklabel(ctx, &lnext, "next.%d");
   2542 		const struct type *subtype =
   2543 			tagged_select_subtype(objtype, _case->type, false);
   2544 		enum match_compat compat = COMPAT_SUBTYPE;
   2545 		if (subtype) {
   2546 			gen_nested_match_tests(ctx, object,
   2547 				bmatch, bnext, tag, _case->type);
   2548 		} else {
   2549 			assert(type_dealias(_case->type)->storage == STORAGE_TAGGED);
   2550 			assert(tagged_subset_compat(objtype, _case->type));
   2551 			if (tagged_align_compat(objtype, _case->type)) {
   2552 				compat = COMPAT_ALIGNED;
   2553 			} else {
   2554 				compat = COMPAT_MISALIGNED;
   2555 			}
   2556 			const struct type *casetype = type_dealias(_case->type);
   2557 			gen_subset_match_tests(ctx, bmatch, bnext, tag, casetype);
   2558 		}
   2559 
   2560 		push(&ctx->current->body, &lmatch);
   2561 
   2562 		if (!_case->object) {
   2563 			goto next;
   2564 		}
   2565 
   2566 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   2567 		gb->value.kind = GV_TEMP;
   2568 		gb->value.type = _case->type;
   2569 		gb->value.name = gen_name(ctx, "binding.%d");
   2570 		gb->object = _case->object;
   2571 		gb->next = ctx->bindings;
   2572 		ctx->bindings = gb;
   2573 
   2574 		struct qbe_value qv = mklval(ctx, &gb->value);
   2575 		enum qbe_instr alloc = alloc_for_align(_case->type->align);
   2576 		struct qbe_value sz = constl(_case->type->size);
   2577 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   2578 
   2579 		struct qbe_value ptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2580 		struct gen_value src = {
   2581 			.kind = GV_TEMP,
   2582 			.type = _case->type,
   2583 			.name = ptr.name,
   2584 		};
   2585 		struct gen_value load;
   2586 		struct qbe_value offset;
   2587 		switch (compat) {
   2588 		case COMPAT_SUBTYPE:
   2589 			offset = nested_tagged_offset(object.type, _case->type);
   2590 			pushi(ctx->current, &ptr, Q_ADD, &qobject, &offset, NULL);
   2591 			load = gen_load(ctx, src);
   2592 			gen_store(ctx, gb->value, load);
   2593 			break;
   2594 		case COMPAT_ALIGNED:
   2595 			pushi(ctx->current, &ptr, Q_COPY, &qobject, NULL);
   2596 			load = gen_load(ctx, src);
   2597 			gen_store(ctx, gb->value, load);
   2598 			break;
   2599 		case COMPAT_MISALIGNED:
   2600 			;
   2601 			struct expression value = {
   2602 				.type = EXPR_GEN_VALUE,
   2603 				.result = objtype,
   2604 				.user = &object,
   2605 			};
   2606 			struct expression cast = {
   2607 				.type = EXPR_CAST,
   2608 				.result = _case->type,
   2609 				.cast = {
   2610 					.kind = C_CAST,
   2611 					.secondary = _case->type,
   2612 					.value = &value,
   2613 				},
   2614 			};
   2615 			gen_expr_at(ctx, &cast, gb->value);
   2616 			break;
   2617 		}
   2618 
   2619 next:
   2620 		bval = gen_expr_with(ctx, _case->value, out);
   2621 		branch_copyresult(ctx, bval, gvout, out);
   2622 		if (!_case->value->terminates) {
   2623 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2624 		}
   2625 		push(&ctx->current->body, &lnext);
   2626 	}
   2627 
   2628 	if (_default) {
   2629 		bval = gen_expr_with(ctx, _default->value, out);
   2630 		branch_copyresult(ctx, bval, gvout, out);
   2631 	} else {
   2632 		struct qbe_statement labort;
   2633 		mklabel(ctx, &labort, ".%d");
   2634 		push(&ctx->current->body, &labort);
   2635 		gen_fixed_abort(ctx, expr->loc, ABORT_UNREACHABLE);
   2636 	}
   2637 
   2638 	push(&ctx->current->body, &lout);
   2639 	return gvout;
   2640 }
   2641 
   2642 static struct gen_value
   2643 gen_match_with_nullable(struct gen_context *ctx,
   2644 	const struct expression *expr,
   2645 	struct gen_value *out)
   2646 {
   2647 	struct gen_value gvout = gv_void;
   2648 	if (!out) {
   2649 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2650 	}
   2651 
   2652 	struct qbe_statement lout;
   2653 	struct qbe_value bout = mklabel(ctx, &lout, ".%d");
   2654 	struct gen_value object = gen_expr(ctx, expr->match.value);
   2655 	struct qbe_value qobject = mkqval(ctx, &object);
   2656 	struct qbe_value zero = constl(0);
   2657 
   2658 	struct gen_value bval;
   2659 	const struct match_case *_default = NULL;
   2660 	for (const struct match_case *_case = expr->match.cases;
   2661 			_case; _case = _case->next) {
   2662 		if (!_case->type) {
   2663 			_default = _case;
   2664 			continue;
   2665 		}
   2666 
   2667 		struct qbe_statement lmatch, lnext;
   2668 		struct qbe_value cmpres = mkqtmp(ctx, &qbe_word, ".%d");
   2669 		struct qbe_value bmatch = mklabel(ctx, &lmatch, "matches.%d");
   2670 		struct qbe_value bnext = mklabel(ctx, &lnext, "next.%d");
   2671 
   2672 		enum qbe_instr compare;
   2673 		if (_case->type->storage == STORAGE_NULL) {
   2674 			compare = Q_CEQL;
   2675 		} else {
   2676 			compare = Q_CNEL;
   2677 		}
   2678 		pushi(ctx->current, &cmpres, compare, &qobject, &zero, NULL);
   2679 		pushi(ctx->current, NULL, Q_JNZ, &cmpres, &bmatch, &bnext, NULL);
   2680 
   2681 		push(&ctx->current->body, &lmatch);
   2682 
   2683 		if (!_case->object) {
   2684 			goto next;
   2685 		}
   2686 
   2687 		struct gen_binding *gb = xcalloc(1, sizeof(struct gen_binding));
   2688 		gb->value = mkgtemp(ctx, _case->type, "binding.%d");
   2689 		gb->object = _case->object;
   2690 		gb->next = ctx->bindings;
   2691 		ctx->bindings = gb;
   2692 
   2693 		enum qbe_instr store = store_for_type(ctx, _case->type);
   2694 		enum qbe_instr alloc = alloc_for_align(_case->type->align);
   2695 		struct qbe_value qv = mkqval(ctx, &gb->value);
   2696 		struct qbe_value sz = constl(_case->type->size);
   2697 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   2698 		pushi(ctx->current, NULL, store, &qobject, &qv, NULL);
   2699 
   2700 next:
   2701 		bval = gen_expr_with(ctx, _case->value, out);
   2702 		branch_copyresult(ctx, bval, gvout, out);
   2703 		if (!_case->value->terminates) {
   2704 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2705 		}
   2706 		push(&ctx->current->body, &lnext);
   2707 	}
   2708 
   2709 	if (_default) {
   2710 		bval = gen_expr_with(ctx, _default->value, out);
   2711 		branch_copyresult(ctx, bval, gvout, out);
   2712 		if (!_default->value->terminates) {
   2713 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2714 		}
   2715 	}
   2716 
   2717 	struct qbe_statement labort;
   2718 	mklabel(ctx, &labort, ".%d");
   2719 	push(&ctx->current->body, &labort);
   2720 	gen_fixed_abort(ctx, expr->loc, ABORT_UNREACHABLE);
   2721 
   2722 	push(&ctx->current->body, &lout);
   2723 	return gvout;
   2724 }
   2725 
   2726 static struct gen_value
   2727 gen_expr_match_with(struct gen_context *ctx,
   2728 	const struct expression *expr,
   2729 	struct gen_value *out)
   2730 {
   2731 	const struct type *objtype = expr->match.value->result;
   2732 	switch (type_dealias(objtype)->storage) {
   2733 	case STORAGE_POINTER:
   2734 		return gen_match_with_nullable(ctx, expr, out);
   2735 	case STORAGE_TAGGED:
   2736 		return gen_match_with_tagged(ctx, expr, out);
   2737 	default: abort(); // Invariant
   2738 	}
   2739 }
   2740 
   2741 static struct gen_value
   2742 gen_expr_measure(struct gen_context *ctx, const struct expression *expr)
   2743 {
   2744 	size_t len;
   2745 	struct gen_value gv, temp;
   2746 	const struct type *type;
   2747 	const struct expression *value = expr->measure.value;
   2748 	switch (expr->measure.op) {
   2749 	case M_LEN:
   2750 		type = type_dealias(type_dereference(value->result));
   2751 		switch (type->storage) {
   2752 		case STORAGE_ARRAY:
   2753 			len = type->array.length;
   2754 			assert(len != SIZE_UNDEFINED);
   2755 			return (struct gen_value){
   2756 				.kind = GV_CONST,
   2757 				.type = &builtin_type_size,
   2758 				.lval = len,
   2759 			};
   2760 		case STORAGE_SLICE:
   2761 		case STORAGE_STRING:
   2762 			gv = gen_expr(ctx, value);
   2763 			gv = gen_autoderef(ctx, gv);
   2764 			temp = mkgtemp(ctx, &builtin_type_size, ".%d");
   2765 			struct qbe_value qv = mkqval(ctx, &gv),
   2766 				qtemp = mkqval(ctx, &temp),
   2767 				offs = constl(builtin_type_size.size);
   2768 			enum qbe_instr load = load_for_type(ctx,
   2769 				&builtin_type_size);
   2770 			pushi(ctx->current, &qtemp, Q_ADD, &qv, &offs, NULL);
   2771 			pushi(ctx->current, &qtemp, load, &qtemp, NULL);
   2772 			return temp;
   2773 		default:
   2774 			abort(); // Invariant
   2775 		}
   2776 		break;
   2777 	case M_SIZE:
   2778 		return (struct gen_value){
   2779 			.kind = GV_CONST,
   2780 			.type = &builtin_type_size,
   2781 			.lval = expr->measure.dimensions.size,
   2782 		};
   2783 	case M_OFFSET:
   2784 		if (expr->measure.value->access.type == ACCESS_FIELD) {
   2785 			return (struct gen_value){
   2786 				.kind = GV_CONST,
   2787 				.type = &builtin_type_size,
   2788 				.lval = expr->measure.value->access.field->offset,
   2789 			};
   2790 		} else {
   2791 			assert(expr->measure.value->access.type == ACCESS_TUPLE);
   2792 			return (struct gen_value){
   2793 				.kind = GV_CONST,
   2794 				.type = &builtin_type_size,
   2795 				.lval = expr->measure.value->access.tvalue->offset,
   2796 			};
   2797 		}
   2798 	}
   2799 	abort(); // Invariant
   2800 }
   2801 
   2802 static struct gen_value
   2803 gen_expr_return(struct gen_context *ctx, const struct expression *expr)
   2804 {
   2805 	struct gen_value ret = gen_expr(ctx, expr->_return.value);
   2806 	for (struct gen_scope *scope = ctx->scope; scope; scope = scope->parent) {
   2807 		gen_defers(ctx, scope);
   2808 	}
   2809 	if (type_dealias(ret.type)->storage == STORAGE_VOID) {
   2810 		pushi(ctx->current, NULL, Q_RET, NULL);
   2811 	} else {
   2812 		struct qbe_value qret = mkqval(ctx, &ret);
   2813 		pushi(ctx->current, NULL, Q_RET, &qret, NULL);
   2814 	}
   2815 	return gv_void;
   2816 }
   2817 
   2818 static void
   2819 gen_expr_struct_at(struct gen_context *ctx,
   2820 	const struct expression *expr,
   2821 	struct gen_value out)
   2822 {
   2823 	// TODO: Merge me into constant expressions
   2824 	struct qbe_value base = mkqval(ctx, &out);
   2825 
   2826 	if (expr->_struct.autofill) {
   2827 		struct qbe_value rtfunc = mkrtfunc(ctx, "rt.memset");
   2828 		struct qbe_value size =
   2829 			constl(expr->result->size), zero = constl(0);
   2830 		struct qbe_value base = mklval(ctx, &out);
   2831 		pushi(ctx->current, NULL, Q_CALL, &rtfunc,
   2832 			&base, &zero, &size, NULL);
   2833 	}
   2834 
   2835 	struct gen_value ftemp = mkgtemp(ctx, &builtin_type_void, "field.%d");
   2836 	for (const struct expr_struct_field *field = expr->_struct.fields;
   2837 			field; field = field->next) {
   2838 		if (!field->value) {
   2839 			assert(expr->_struct.autofill);
   2840 			continue;
   2841 		}
   2842 
   2843 		struct qbe_value offs = constl(field->field->offset);
   2844 		ftemp.type = field->value->result;
   2845 		struct qbe_value ptr = mklval(ctx, &ftemp);
   2846 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   2847 		gen_expr_at(ctx, field->value, ftemp);
   2848 	}
   2849 }
   2850 
   2851 static struct gen_value
   2852 gen_expr_switch_with(struct gen_context *ctx,
   2853 	const struct expression *expr,
   2854 	struct gen_value *out)
   2855 {
   2856 	struct gen_value gvout = gv_void;
   2857 	if (!out) {
   2858 		gvout = mkgtemp(ctx, expr->result, ".%d");
   2859 	}
   2860 
   2861 	struct qbe_statement lout;
   2862 	struct qbe_value bout = mklabel(ctx, &lout, ".%d");
   2863 	struct gen_value value = gen_expr(ctx, expr->_switch.value);
   2864 
   2865 	struct gen_value bval;
   2866 	const struct switch_case *_default = NULL;
   2867 	for (const struct switch_case *_case = expr->_switch.cases;
   2868 			_case; _case = _case->next) {
   2869 		if (!_case->options) {
   2870 			_default = _case;
   2871 			continue;
   2872 		}
   2873 
   2874 		struct qbe_statement lmatch, lnextcase;
   2875 		struct qbe_value bmatch = mklabel(ctx, &lmatch, "matches.%d");
   2876 		struct qbe_value bnextcase = mklabel(ctx, &lnextcase, "next.%d");
   2877 
   2878 		for (struct case_option *opt = _case->options;
   2879 				opt; opt = opt->next) {
   2880 			struct qbe_statement lnextopt;
   2881 			struct qbe_value bnextopt = mklabel(ctx, &lnextopt, ".%d");
   2882 			struct gen_value test = gen_expr_const(ctx, opt->value);
   2883 			struct expression lvalue = {
   2884 				.type = EXPR_GEN_VALUE,
   2885 				.result = value.type,
   2886 				.user = &value,
   2887 			}, rvalue = {
   2888 				.type = EXPR_GEN_VALUE,
   2889 				.result = test.type,
   2890 				.user = &test,
   2891 			}, compare = {
   2892 				.type = EXPR_BINARITHM,
   2893 				.result = &builtin_type_bool,
   2894 				.binarithm = {
   2895 					.op = BIN_LEQUAL,
   2896 					.lvalue = &lvalue,
   2897 					.rvalue = &rvalue,
   2898 				},
   2899 			};
   2900 			struct gen_value match = gen_expr(ctx, &compare);
   2901 			struct qbe_value cond = mkqval(ctx, &match);
   2902 			pushi(ctx->current, NULL, Q_JNZ,
   2903 				&cond, &bmatch, &bnextopt, NULL);
   2904 			push(&ctx->current->body, &lnextopt);
   2905 		}
   2906 
   2907 		pushi(ctx->current, NULL, Q_JMP, &bnextcase, NULL);
   2908 		push(&ctx->current->body, &lmatch);
   2909 		bval = gen_expr_with(ctx, _case->value, out);
   2910 		branch_copyresult(ctx, bval, gvout, out);
   2911 		if (!_case->value->terminates) {
   2912 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2913 		}
   2914 		push(&ctx->current->body, &lnextcase);
   2915 	}
   2916 
   2917 	if (_default) {
   2918 		bval = gen_expr_with(ctx, _default->value, out);
   2919 		branch_copyresult(ctx, bval, gvout, out);
   2920 		if (!_default->value->terminates) {
   2921 			pushi(ctx->current, NULL, Q_JMP, &bout, NULL);
   2922 		}
   2923 	}
   2924 
   2925 	struct qbe_statement labort;
   2926 	mklabel(ctx, &labort, ".%d");
   2927 	push(&ctx->current->body, &labort);
   2928 	gen_fixed_abort(ctx, expr->loc, ABORT_UNREACHABLE);
   2929 
   2930 	push(&ctx->current->body, &lout);
   2931 	return gvout;
   2932 }
   2933 
   2934 static void
   2935 gen_expr_slice_at(struct gen_context *ctx,
   2936 	const struct expression *expr,
   2937 	struct gen_value out)
   2938 {
   2939 	struct gen_value object = gen_expr(ctx, expr->slice.object);
   2940 	object = gen_autoderef(ctx, object);
   2941 	const struct type *srctype = type_dealias(object.type);
   2942 
   2943 	bool check_bounds = !expr->slice.bounds_checked;
   2944 	struct gen_value length;
   2945 	struct qbe_value qlength;
   2946 	struct qbe_value qbase;
   2947 	struct qbe_value qobject = mkqval(ctx, &object);
   2948 	struct qbe_value offset = constl(builtin_type_size.size);
   2949 	struct qbe_value qptr = mkqtmp(ctx, ctx->arch.ptr, ".%d");
   2950 	switch (srctype->storage) {
   2951 	case STORAGE_ARRAY:
   2952 		if (srctype->array.length != SIZE_UNDEFINED) {
   2953 			length = (struct gen_value){
   2954 				.kind = GV_CONST,
   2955 				.type = &builtin_type_size,
   2956 				.lval = srctype->array.length,
   2957 			};
   2958 			qlength = mkqval(ctx, &length);
   2959 		} else {
   2960 			assert(expr->slice.end);
   2961 			check_bounds = false;
   2962 		}
   2963 		qbase = mkqval(ctx, &object);
   2964 		break;
   2965 	case STORAGE_SLICE:
   2966 		qbase = mkqtmp(ctx, ctx->arch.sz, "base.%d");
   2967 		enum qbe_instr load = load_for_type(ctx, &builtin_type_size);
   2968 		pushi(ctx->current, &qbase, load, &qobject, NULL);
   2969 		length = mkgtemp(ctx, &builtin_type_size, "len.%d");
   2970 		qlength = mkqval(ctx, &length);
   2971 		pushi(ctx->current, &qptr, Q_ADD, &qobject, &offset, NULL);
   2972 		pushi(ctx->current, &qlength, load, &qptr, NULL);
   2973 		break;
   2974 	default: abort(); // Invariant
   2975 	}
   2976 
   2977 	struct gen_value start;
   2978 	if (expr->slice.start) {
   2979 		start = gen_expr(ctx, expr->slice.start);
   2980 	} else {
   2981 		start = (struct gen_value){
   2982 			.kind = GV_CONST,
   2983 			.type = &builtin_type_size,
   2984 			.lval = 0,
   2985 		};
   2986 	}
   2987 
   2988 	struct gen_value end;
   2989 	if (expr->slice.end) {
   2990 		end = gen_expr(ctx, expr->slice.end);
   2991 	} else {
   2992 		end = length;
   2993 	}
   2994 
   2995 	struct qbe_value qstart = mkqval(ctx, &start);
   2996 	struct qbe_value qend = mkqval(ctx, &end);
   2997 
   2998 	if (check_bounds) {
   2999 		struct qbe_value start_oob = mkqtmp(ctx, &qbe_word, ".%d");
   3000 		struct qbe_value end_oob = mkqtmp(ctx, &qbe_word, ".%d");
   3001 		struct qbe_value startend_oob = mkqtmp(ctx, &qbe_word, ".%d");
   3002 		struct qbe_value valid = mkqtmp(ctx, &qbe_word, ".%d");
   3003 		pushi(ctx->current, &start_oob, Q_CULEL, &qstart, &qlength, NULL);
   3004 		pushi(ctx->current, &end_oob, Q_CULEL, &qend, &qlength, NULL);
   3005 		pushi(ctx->current, &valid, Q_AND, &start_oob, &end_oob, NULL);
   3006 		pushi(ctx->current, &startend_oob, Q_CULEL, &qstart, &qend, NULL);
   3007 		pushi(ctx->current, &valid, Q_AND, &valid, &startend_oob, NULL);
   3008 
   3009 		struct qbe_statement linvalid, lvalid;
   3010 		struct qbe_value binvalid = mklabel(ctx, &linvalid, ".%d");
   3011 		struct qbe_value bvalid = mklabel(ctx, &lvalid, ".%d");
   3012 
   3013 		pushi(ctx->current, NULL, Q_JNZ, &valid, &bvalid, &binvalid, NULL);
   3014 		push(&ctx->current->body, &linvalid);
   3015 		gen_fixed_abort(ctx, expr->loc, ABORT_OOB);
   3016 		push(&ctx->current->body, &lvalid);
   3017 	}
   3018 
   3019 	struct qbe_value isz = constl(srctype->array.members->size);
   3020 
   3021 	struct qbe_value qout = mkqval(ctx, &out);
   3022 	struct qbe_value data = mkqtmp(ctx, ctx->arch.ptr, "data.%d");
   3023 	pushi(ctx->current, &data, Q_MUL, &qstart, &isz, NULL);
   3024 	pushi(ctx->current, &data, Q_ADD, &qbase, &data, NULL);
   3025 
   3026 	struct qbe_value newlen = mkqtmp(ctx, ctx->arch.sz, "newlen.%d");
   3027 	pushi(ctx->current, &newlen, Q_SUB, &qend, &qstart, NULL);
   3028 	struct qbe_value newcap = mkqtmp(ctx, ctx->arch.sz, "newcap.%d");
   3029 	if (check_bounds) {
   3030 		pushi(ctx->current, &newcap, Q_SUB, &qlength, &qstart, NULL);
   3031 	} else {
   3032 		pushi(ctx->current, &newcap, Q_COPY, &newlen, NULL);
   3033 	}
   3034 
   3035 	enum qbe_instr store = store_for_type(ctx, &builtin_type_size);
   3036 	pushi(ctx->current, NULL, store, &data, &qout, NULL);
   3037 	pushi(ctx->current, &qptr, Q_ADD, &qout, &offset, NULL);
   3038 	pushi(ctx->current, NULL, store, &newlen, &qptr, NULL);
   3039 	offset = constl(builtin_type_size.size * 2);
   3040 	pushi(ctx->current, &qptr, Q_ADD, &qout, &offset, NULL);
   3041 	pushi(ctx->current, NULL, store, &newcap, &qptr, NULL);
   3042 }
   3043 
   3044 static void
   3045 gen_expr_tuple_at(struct gen_context *ctx,
   3046 	const struct expression *expr,
   3047 	struct gen_value out)
   3048 {
   3049 	// TODO: Merge me into constant expressions
   3050 	struct qbe_value base = mkqval(ctx, &out);
   3051 
   3052 	const struct type *type = type_dealias(expr->result);
   3053 	struct gen_value vtemp = mkgtemp(ctx, &builtin_type_void, "value.%d");
   3054 	const struct expression_tuple *value = &expr->tuple;
   3055 	for (const struct type_tuple *tuple = &type->tuple;
   3056 			tuple; tuple = tuple->next) {
   3057 		struct qbe_value offs = constl(tuple->offset);
   3058 		vtemp.type = value->value->result;
   3059 		struct qbe_value ptr = mklval(ctx, &vtemp);
   3060 		pushi(ctx->current, &ptr, Q_ADD, &base, &offs, NULL);
   3061 		gen_expr_at(ctx, value->value, vtemp);
   3062 		value = value->next;
   3063 	}
   3064 }
   3065 
   3066 static struct gen_value
   3067 gen_expr_unarithm(struct gen_context *ctx,
   3068 	const struct expression *expr)
   3069 {
   3070 	struct gen_value val, temp;
   3071 	struct qbe_value qval, qtmp;
   3072 	const struct expression *operand = expr->unarithm.operand;
   3073 	switch (expr->unarithm.op) {
   3074 	case UN_ADDRESS:
   3075 		if (operand->type == EXPR_ACCESS) {
   3076 			val = gen_expr_access_addr(ctx, operand);
   3077 			val.type = expr->result;
   3078 			return val;
   3079 		}
   3080 		struct gen_value val = mkgtemp(ctx, operand->result, ".%d");
   3081 		struct qbe_value qv = mklval(ctx, &val);
   3082 		struct qbe_value sz = constl(val.type->size);
   3083 		enum qbe_instr alloc = alloc_for_align(val.type->align);
   3084 		pushprei(ctx->current, &qv, alloc, &sz, NULL);
   3085 		gen_expr_at(ctx, operand, val);
   3086 		val.type = expr->result;
   3087 		return val;
   3088 	case UN_DEREF:
   3089 		val = gen_expr(ctx, operand);
   3090 		assert(type_dealias(val.type)->storage == STORAGE_POINTER);
   3091 		val.type = type_dealias(val.type)->pointer.referent;
   3092 		return gen_load(ctx, val);
   3093 	case UN_BNOT:
   3094 		val = gen_expr(ctx, operand);
   3095 		temp = mkgtemp(ctx, operand->result, ".%d");
   3096 		qval = mkqval(ctx, &val), qtmp = mkqval(ctx, &temp);
   3097 		struct qbe_value ones = constl((uint64_t)-1);
   3098 		pushi(ctx->current, &qtmp, Q_XOR, &qval, &ones, NULL);
   3099 		return temp;
   3100 	case UN_LNOT:
   3101 		val = gen_expr(ctx, operand);
   3102 		temp = mkgtemp(ctx, operand->result, ".%d");
   3103 		qval = mkqval(ctx, &val), qtmp = mkqval(ctx, &temp);
   3104 		struct qbe_value zerow = constw(0);
   3105 		pushi(ctx->current, &qtmp, Q_CEQW, &qval, &zerow, NULL);
   3106 		return temp;
   3107 	case UN_MINUS:
   3108 		val = gen_expr(ctx, operand);
   3109 		temp = mkgtemp(ctx, operand->result, ".%d");
   3110 		qval = mkqval(ctx, &val), qtmp = mkqval(ctx, &temp);
   3111 		pushi(ctx->current, &qtmp, Q_NEG, &qval, NULL);
   3112 		return temp;
   3113 	case UN_PLUS:
   3114 		return gen_expr(ctx, operand);
   3115 	}
   3116 	abort(); // Invariant
   3117 }
   3118 
   3119 static struct gen_value
   3120 gen_expr_vaarg(struct gen_context *ctx,
   3121 	const struct expression *expr)
   3122 {
   3123 	// XXX: qbe only supports variadic base types, should check for this
   3124 	struct gen_value result = mkgtemp(ctx, expr->result, ".%d");
   3125 	struct qbe_value qresult = mkqval(ctx, &result);
   3126 	struct gen_value ap = gen_expr(ctx, expr->vaarg.ap);
   3127 	struct qbe_value qap = mkqval(ctx, &ap);
   3128 	pushi(ctx->current, &qresult, Q_VAARG, &qap, NULL);
   3129 	return result;
   3130 }
   3131 
   3132 static void
   3133 gen_expr_vastart_at(struct gen_context *ctx,
   3134 	const struct expression *expr,
   3135 	struct gen_value out)
   3136 {
   3137 	struct qbe_value base = mklval(ctx, &out);
   3138 	pushi(ctx->current, NULL, Q_VASTART, &base, NULL);
   3139 }
   3140 
   3141 static struct gen_value
   3142 gen_expr(struct gen_context *ctx, const struct expression *expr)
   3143 {
   3144 	switch ((int)expr->type) {
   3145 	case EXPR_ACCESS:
   3146 		return gen_expr_access(ctx, expr);
   3147 	case EXPR_ALLOC:
   3148 		return gen_expr_alloc_with(ctx, expr, NULL);
   3149 	case EXPR_APPEND:
   3150 		return gen_expr_append(ctx, expr);
   3151 	case EXPR_ASSERT:
   3152 		return gen_expr_assert(ctx, expr);
   3153 	case EXPR_ASSIGN:
   3154 		return gen_expr_assign(ctx, expr);
   3155 	case EXPR_BINARITHM:
   3156 		return gen_expr_binarithm(ctx, expr);
   3157 	case EXPR_BINDING:
   3158 		return gen_expr_binding(ctx, expr);
   3159 	case EXPR_BREAK:
   3160 	case EXPR_CONTINUE:
   3161 	case EXPR_YIELD:
   3162 		return gen_expr_control(ctx, expr);
   3163 	case EXPR_CALL:
   3164 		return gen_expr_call(ctx, expr);
   3165 	case EXPR_CAST:
   3166 		return gen_expr_cast(ctx, expr);
   3167 	case EXPR_COMPOUND:
   3168 		return gen_expr_compound_with(ctx, expr, NULL);
   3169 	case EXPR_CONSTANT:
   3170 		return gen_expr_const(ctx, expr);
   3171 	case EXPR_DEFER:
   3172 		return gen_expr_defer(ctx, expr);
   3173 	case EXPR_DELETE:
   3174 		return gen_expr_delete(ctx, expr);
   3175 	case EXPR_FOR:
   3176 		return gen_expr_for(ctx, expr);
   3177 	case EXPR_FREE:
   3178 		return gen_expr_free(ctx, expr);
   3179 	case EXPR_IF:
   3180 		return gen_expr_if_with(ctx, expr, NULL);
   3181 	case EXPR_INSERT:
   3182 		return gen_expr_insert(ctx, expr);
   3183 	case EXPR_MATCH:
   3184 		return gen_expr_match_with(ctx, expr, NULL);
   3185 	case EXPR_MEASURE:
   3186 		return gen_expr_measure(ctx, expr);
   3187 	case EXPR_PROPAGATE:
   3188 		assert(0); // Lowered in check (for now?)
   3189 	case EXPR_RETURN:
   3190 		return gen_expr_return(ctx, expr);
   3191 	case EXPR_SWITCH:
   3192 		return gen_expr_switch_with(ctx, expr, NULL);
   3193 	case EXPR_UNARITHM:
   3194 		return gen_expr_unarithm(ctx, expr);
   3195 	case EXPR_VAARG:
   3196 		return gen_expr_vaarg(ctx, expr);
   3197 	case EXPR_VAEND:
   3198 		return gv_void; // no-op
   3199 	case EXPR_SLICE:
   3200 	case EXPR_STRUCT:
   3201 	case EXPR_TUPLE:
   3202 	case EXPR_VASTART:
   3203 		break; // Prefers -at style
   3204 	// gen-specific psuedo-expressions
   3205 	case EXPR_GEN_VALUE:
   3206 		return *(struct gen_value *)expr->user;
   3207 	}
   3208 
   3209 	struct gen_value out = mkgtemp(ctx, expr->result, "object.%d");
   3210 	struct qbe_value base = mkqval(ctx, &out);
   3211 	struct qbe_value sz = constl(expr->result->size);
   3212 	enum qbe_instr alloc = alloc_for_align(expr->result->align);
   3213 	pushprei(ctx->current, &base, alloc, &sz, NULL);
   3214 	gen_expr_at(ctx, expr, out);
   3215 	return out;
   3216 }
   3217 
   3218 static void
   3219 gen_expr_at(struct gen_context *ctx,
   3220 	const struct expression *expr,
   3221 	struct gen_value out)
   3222 {
   3223 	assert(out.kind != GV_CONST);
   3224 
   3225 	switch (expr->type) {
   3226 	case EXPR_ALLOC:
   3227 		gen_expr_alloc_with(ctx, expr, &out);
   3228 		return;
   3229 	case EXPR_CAST:
   3230 		gen_expr_cast_at(ctx, expr, out);
   3231 		return;
   3232 	case EXPR_COMPOUND:
   3233 		gen_expr_compound_with(ctx, expr, &out);
   3234 		return;
   3235 	case EXPR_CONSTANT:
   3236 		gen_expr_const_at(ctx, expr, out);
   3237 		return;
   3238 	case EXPR_IF:
   3239 		gen_expr_if_with(ctx, expr, &out);
   3240 		return;
   3241 	case EXPR_MATCH:
   3242 		gen_expr_match_with(ctx, expr, &out);
   3243 		return;
   3244 	case EXPR_SLICE:
   3245 		gen_expr_slice_at(ctx, expr, out);
   3246 		return;
   3247 	case EXPR_STRUCT:
   3248 		gen_expr_struct_at(ctx, expr, out);
   3249 		return;
   3250 	case EXPR_SWITCH:
   3251 		gen_expr_switch_with(ctx, expr, &out);
   3252 		return;
   3253 	case EXPR_TUPLE:
   3254 		gen_expr_tuple_at(ctx, expr, out);
   3255 		return;
   3256 	case EXPR_VASTART:
   3257 		gen_expr_vastart_at(ctx, expr, out);
   3258 		return;
   3259 	default:
   3260 		break; // Prefers non-at style
   3261 	}
   3262 
   3263 	struct gen_value result = gen_expr(ctx, expr);
   3264 	if (!expr->terminates) {
   3265 		gen_store(ctx, out, result);
   3266 	}
   3267 }
   3268 
   3269 static struct gen_value
   3270 gen_expr_with(struct gen_context *ctx,
   3271 	const struct expression *expr,
   3272 	struct gen_value *out)
   3273 {
   3274 	if (out) {
   3275 		gen_expr_at(ctx, expr, *out);
   3276 		return *out;
   3277 	}
   3278 	return gen_expr(ctx, expr);
   3279 }
   3280 
   3281 static struct qbe_data_item *gen_data_item(struct gen_context *,
   3282 	struct expression *, struct qbe_data_item *);
   3283 
   3284 static void
   3285 gen_function_decl(struct gen_context *ctx, const struct declaration *decl)
   3286 {
   3287 	const struct function_decl *func = &decl->func;
   3288 	const struct type *fntype = func->type;
   3289 	if (func->body == NULL) {
   3290 		return; // Prototype
   3291 	}
   3292 
   3293 	struct qbe_def *qdef = xcalloc(1, sizeof(struct qbe_def));
   3294 	qdef->kind = Q_FUNC;
   3295 	qdef->exported = decl->exported;
   3296 	ctx->current = &qdef->func;
   3297 
   3298 	if (func->flags & FN_TEST) {
   3299 		qdef->name = gen_name(ctx, "testfunc.%d");
   3300 	} else if (func->flags & FN_INIT) {
   3301 		qdef->name = gen_name(ctx, "initfunc.%d");
   3302 	} else if (func->flags & FN_FINI) {
   3303 		qdef->name = gen_name(ctx, "finifunc.%d");
   3304 	} else {
   3305 		qdef->name = decl->symbol ? xstrdup(decl->symbol)
   3306 			: ident_to_sym(&decl->ident);
   3307 	}
   3308 
   3309 	struct qbe_statement start_label = {0};
   3310 	mklabel(ctx, &start_label, "start.%d");
   3311 	push(&qdef->func.prelude, &start_label);
   3312 
   3313 	if (type_dealias(fntype->func.result)->storage != STORAGE_VOID) {
   3314 		qdef->func.returns = qtype_lookup(
   3315 			ctx, fntype->func.result, false);
   3316 	} else {
   3317 		qdef->func.returns = &qbe_void;
   3318 	}
   3319 	if (fntype->func.variadism == VARIADISM_C) {
   3320 		qdef->func.variadic = true;
   3321 	}
   3322 
   3323 	struct qbe_func_param *param, **next = &qdef->func.params;
   3324 	for (struct scope_object *obj = decl->func.scope->objects;
   3325 			obj; obj = obj->lnext) {
   3326 		const struct type *type = obj->type;
   3327 		param = *next = xcalloc(1, sizeof(struct qbe_func_param));
   3328 		assert(!obj->ident.ns); // Invariant
   3329 		param->name = xstrdup(obj->ident.name);
   3330 		param->type = qtype_lookup(ctx, type, false);
   3331 
   3332 		struct gen_binding *gb =
   3333 			xcalloc(1, sizeof(struct gen_binding));
   3334 		gb->value.kind = GV_TEMP;
   3335 		gb->value.type = type;
   3336 		gb->object = obj;
   3337 		if (type_is_aggregate(type)) {
   3338 			// No need to copy to stack
   3339 			gb->value.name = xstrdup(param->name);
   3340 		} else {
   3341 			gb->value.name = gen_name(ctx, "param.%d");
   3342 
   3343 			struct qbe_value qv = mklval(ctx, &gb->value);
   3344 			struct qbe_value sz = constl(type->size);
   3345 			enum qbe_instr alloc = alloc_for_align(type->align);
   3346 			pushprei(ctx->current, &qv, alloc, &sz, NULL);
   3347 			struct gen_value src = {
   3348 				.kind = GV_TEMP,
   3349 				.type = type,
   3350 				.name = param->name,
   3351 			};
   3352 			gen_store(ctx, gb->value, src);
   3353 		}
   3354 
   3355 		gb->next = ctx->bindings;
   3356 		ctx->bindings = gb;
   3357 		next = &param->next;
   3358 	}
   3359 
   3360 	pushl(&qdef->func, &ctx->id, "body.%d");
   3361 	struct gen_value ret = gen_expr(ctx, decl->func.body);
   3362 
   3363 	if (decl->func.body->terminates) {
   3364 		// XXX: This is a bit hacky, to appease qbe
   3365 		size_t ln = ctx->current->body.ln;
   3366 		struct qbe_statement *last = &ctx->current->body.stmts[ln - 1];
   3367 		if (last->type != Q_INSTR || last->instr != Q_RET) {
   3368 			pushi(ctx->current, NULL, Q_RET, NULL);
   3369 		}
   3370 	} else if (type_dealias(fntype->func.result)->storage != STORAGE_VOID) {
   3371 		struct qbe_value qret = mkqval(ctx, &ret);
   3372 		pushi(ctx->current, NULL, Q_RET, &qret, NULL);
   3373 	} else {
   3374 		pushi(ctx->current, NULL, Q_RET, NULL);
   3375 	}
   3376 
   3377 	qbe_append_def(ctx->out, qdef);
   3378 
   3379 	if (func->flags & FN_INIT) {
   3380 		struct qbe_def *init = xcalloc(1, sizeof *init);
   3381 		init->kind = Q_DATA;
   3382 		init->exported = false;
   3383 		init->data.align = 8;
   3384 		init->data.section = ".init_array";
   3385 		init->data.secflags = NULL;
   3386 
   3387 		size_t n = snprintf(NULL, 0, ".init.%s", qdef->name);
   3388 		init->name = xcalloc(n + 1, 1);
   3389 		snprintf(init->name, n + 1, ".init.%s", qdef->name);
   3390 
   3391 		struct qbe_data_item dataitem = {
   3392 			.type = QD_VALUE,
   3393 			.value = {
   3394 				.kind = QV_GLOBAL,
   3395 				.type = &qbe_long,
   3396 				.name = xstrdup(qdef->name),
   3397 			},
   3398 			.next = NULL,
   3399 		};
   3400 		init->data.items = dataitem;
   3401 
   3402 		qbe_append_def(ctx->out, init);
   3403 	}
   3404 
   3405 	if (func->flags & FN_FINI) {
   3406 		struct qbe_def *fini = xcalloc(1, sizeof *fini);
   3407 		fini->kind = Q_DATA;
   3408 		fini->exported = false;
   3409 		fini->data.align = 8;
   3410 		fini->data.section = ".fini_array";
   3411 		fini->data.secflags = NULL;
   3412 
   3413 		size_t n = snprintf(NULL, 0, ".fini.%s", qdef->name);
   3414 		fini->name = xcalloc(n + 1, 1);
   3415 		snprintf(fini->name, n + 1, ".fini.%s", qdef->name);
   3416 
   3417 		struct qbe_data_item dataitem = {
   3418 			.type = QD_VALUE,
   3419 			.value = {
   3420 				.kind = QV_GLOBAL,
   3421 				.type = &qbe_long,
   3422 				.name = xstrdup(qdef->name),
   3423 			},
   3424 			.next = NULL,
   3425 		};
   3426 		fini->data.items = dataitem;
   3427 
   3428 		qbe_append_def(ctx->out, fini);
   3429 	}
   3430 
   3431 	if (func->flags & FN_TEST) {
   3432 		struct qbe_def *test = xcalloc(1, sizeof *test);
   3433 		test->kind = Q_DATA;
   3434 		test->exported = false;
   3435 		test->data.align = 8;
   3436 		test->data.section = ".test_array";
   3437 		test->data.secflags = "aw";
   3438 
   3439 		size_t n = snprintf(NULL, 0, ".test.%s", qdef->name);
   3440 		test->name = xcalloc(n + 1, 1);
   3441 		snprintf(test->name, n + 1, ".test.%s", qdef->name);
   3442 
   3443 		char *ident = identifier_unparse(&decl->ident);
   3444 
   3445 		struct qbe_data_item *dataitem = &test->data.items;
   3446 		struct expression expr = {
   3447 			.type = EXPR_CONSTANT,
   3448 			.result = &builtin_type_str,
   3449 			.constant = {
   3450 				.object = NULL,
   3451 				.string = {
   3452 					.value = ident,
   3453 					.len = strlen(ident),
   3454 				},
   3455 			},
   3456 		};
   3457 		dataitem = gen_data_item(ctx, &expr, dataitem);
   3458 
   3459 		struct qbe_data_item *next = xcalloc(1, sizeof *next);
   3460 		next->type = QD_VALUE;
   3461 		next->value.kind = QV_GLOBAL;
   3462 		next->value.type = &qbe_long;
   3463 		next->value.name = xstrdup(qdef->name);
   3464 		next->next = NULL;
   3465 		dataitem->next = next;
   3466 
   3467 		qbe_append_def(ctx->out, test);
   3468 	}
   3469 
   3470 	ctx->current = NULL;
   3471 }
   3472 
   3473 static struct qbe_data_item *
   3474 gen_data_item(struct gen_context *ctx, struct expression *expr,
   3475 	struct qbe_data_item *item)
   3476 {
   3477 	assert(expr->type == EXPR_CONSTANT);
   3478 
   3479 	struct qbe_def *def;
   3480 	const struct expression_constant *constant = &expr->constant;
   3481 	const struct type *type = type_dealias(expr->result);
   3482 	type = lower_const(type, NULL);
   3483 	if (constant->object) {
   3484 		item->type = QD_SYMOFFS;
   3485 		item->sym = ident_to_sym(&constant->object->ident);
   3486 		item->offset = constant->ival;
   3487 		return item;
   3488 	}
   3489 
   3490 	switch (type->storage) {
   3491 	case STORAGE_I8:
   3492 	case STORAGE_U8:
   3493 	case STORAGE_CHAR:
   3494 		item->type = QD_VALUE;
   3495 		item->value = constw((uint8_t)constant->uval);
   3496 		item->value.type = &qbe_byte;
   3497 		break;
   3498 	case STORAGE_BOOL:
   3499 		item->type = QD_VALUE;
   3500 		item->value = constw(constant->bval ? 1 : 0);
   3501 		item->value.type = &qbe_byte;
   3502 		break;
   3503 	case STORAGE_I16:
   3504 	case STORAGE_U16:
   3505 		item->type = QD_VALUE;
   3506 		item->value = constw((uint16_t)constant->uval);
   3507 		item->value.type = &qbe_half;
   3508 		break;
   3509 	case STORAGE_I32:
   3510 	case STORAGE_U32:
   3511 	case STORAGE_INT:
   3512 	case STORAGE_UINT:
   3513 	case STORAGE_RUNE:
   3514 		item->type = QD_VALUE;
   3515 		item->value = constw((uint32_t)constant->uval);
   3516 		break;
   3517 	case STORAGE_U64:
   3518 	case STORAGE_I64:
   3519 	case STORAGE_SIZE:
   3520 		item->type = QD_VALUE;
   3521 		item->value = constl((uint64_t)constant->uval);
   3522 		break;
   3523 	case STORAGE_F32:
   3524 		item->type = QD_VALUE;
   3525 		item->value = consts((float)constant->fval);
   3526 		break;
   3527 	case STORAGE_F64:
   3528 		item->type = QD_VALUE;
   3529 		item->value = constd((double)constant->fval);
   3530 		break;
   3531 	case STORAGE_UINTPTR:
   3532 	case STORAGE_POINTER:
   3533 		assert(expr->type == EXPR_CONSTANT); // TODO?
   3534 		item->type = QD_VALUE;
   3535 		switch (ctx->arch.ptr->stype) {
   3536 		case Q_LONG:
   3537 			item->value = constl((uint64_t)constant->uval);
   3538 			break;
   3539 		default: assert(0);
   3540 		}
   3541 		break;
   3542 	case STORAGE_ARRAY:
   3543 		assert(type->array.length != SIZE_UNDEFINED);
   3544 		size_t n = type->array.length;
   3545 		for (struct array_constant *c = constant->array;
   3546 				c && n; c = c->next ? c->next : c, --n) {
   3547 			item = gen_data_item(ctx, c->value, item);
   3548 			if (n > 1 || c->next) {
   3549 				item->next = xcalloc(1,
   3550 					sizeof(struct qbe_data_item));
   3551 				item = item->next;
   3552 			}
   3553 		}
   3554 		break;
   3555 	case STORAGE_STRING:
   3556 		def = xcalloc(1, sizeof(struct qbe_def));
   3557 		def->name = gen_name(ctx, "strdata.%d");
   3558 		def->kind = Q_DATA;
   3559 		def->data.align = ALIGN_UNDEFINED;
   3560 		def->data.items.type = QD_STRING;
   3561 		def->data.items.str = xcalloc(1, expr->constant.string.len);
   3562 		def->data.items.sz = expr->constant.string.len;
   3563 		memcpy(def->data.items.str, expr->constant.string.value,
   3564 			expr->constant.string.len);
   3565 
   3566 		item->type = QD_VALUE;
   3567 		if (expr->constant.string.len != 0) {
   3568 			qbe_append_def(ctx->out, def);
   3569 			item->value.kind = QV_GLOBAL;
   3570 			item->value.type = &qbe_long;
   3571 			item->value.name = xstrdup(def->name);
   3572 		} else {
   3573 			free(def);
   3574 			item->value = constl(0);
   3575 		}
   3576 
   3577 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3578 		item = item->next;
   3579 		item->type = QD_VALUE;
   3580 		item->value = constl(expr->constant.string.len);
   3581 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3582 		item = item->next;
   3583 		item->type = QD_VALUE;
   3584 		item->value = constl(expr->constant.string.len);
   3585 		break;
   3586 	case STORAGE_SLICE:
   3587 		def = xcalloc(1, sizeof(struct qbe_def));
   3588 		def->name = gen_name(ctx, "sldata.%d");
   3589 		def->kind = Q_DATA;
   3590 		def->data.align = ALIGN_UNDEFINED;
   3591 
   3592 		size_t len = 0;
   3593 		struct qbe_data_item *subitem = &def->data.items;
   3594 		for (struct array_constant *c = constant->array;
   3595 				c; c = c->next) {
   3596 			subitem = gen_data_item(ctx, c->value, subitem);
   3597 			if (c->next) {
   3598 				subitem->next = xcalloc(1,
   3599 					sizeof(struct qbe_data_item));
   3600 				subitem = subitem->next;
   3601 			}
   3602 			++len;
   3603 		}
   3604 
   3605 		item->type = QD_VALUE;
   3606 		if (len != 0) {
   3607 			qbe_append_def(ctx->out, def);
   3608 			item->value.kind = QV_GLOBAL;
   3609 			item->value.type = &qbe_long;
   3610 			item->value.name = xstrdup(def->name);
   3611 		} else {
   3612 			free(def);
   3613 			item->value = constl(0);
   3614 		}
   3615 
   3616 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3617 		item = item->next;
   3618 		item->type = QD_VALUE;
   3619 		item->value = constl(len);
   3620 		item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3621 		item = item->next;
   3622 		item->type = QD_VALUE;
   3623 		item->value = constl(len);
   3624 		break;
   3625 	case STORAGE_STRUCT:
   3626 		for (struct struct_constant *f = constant->_struct;
   3627 				f; f = f->next) {
   3628 			item = gen_data_item(ctx, f->value, item);
   3629 			if (f->next) {
   3630 				const struct struct_field *f1 = f->field;
   3631 				const struct struct_field *f2 = f->next->field;
   3632 				if (f2->offset != f1->offset + f1->type->size) {
   3633 					item->next = xcalloc(1,
   3634 						sizeof(struct qbe_data_item));
   3635 					item = item->next;
   3636 					item->type = QD_ZEROED;
   3637 					item->zeroed = f2->offset -
   3638 						(f1->offset + f1->type->size);
   3639 				}
   3640 
   3641 				item->next = xcalloc(1,
   3642 					sizeof(struct qbe_data_item));
   3643 				item = item->next;
   3644 			} else {
   3645 				const struct struct_field *fi = f->field;
   3646 				if (fi->offset + fi->type->size
   3647 						!= expr->result->size) {
   3648 					item->next = xcalloc(1,
   3649 						sizeof(struct qbe_data_item));
   3650 					item = item->next;
   3651 					item->type = QD_ZEROED;
   3652 					item->zeroed = expr->result->size
   3653 						- (fi->offset + fi->type->size);
   3654 				}
   3655 			}
   3656 		}
   3657 		break;
   3658 	case STORAGE_ENUM:
   3659 		switch (type->alias.type->storage) {
   3660 		case STORAGE_I8:
   3661 		case STORAGE_U8:
   3662 		case STORAGE_CHAR:
   3663 			item->type = QD_VALUE;
   3664 			item->value = constw((uint8_t)constant->uval);
   3665 			item->value.type = &qbe_byte;
   3666 			break;
   3667 		case STORAGE_I16:
   3668 		case STORAGE_U16:
   3669 			item->type = QD_VALUE;
   3670 			item->value = constw((uint16_t)constant->uval);
   3671 			item->value.type = &qbe_half;
   3672 			break;
   3673 		case STORAGE_I32:
   3674 		case STORAGE_U32:
   3675 		case STORAGE_INT: // XXX: arch
   3676 		case STORAGE_UINT:
   3677 			item->type = QD_VALUE;
   3678 			item->value = constw((uint32_t)constant->uval);
   3679 			break;
   3680 		case STORAGE_U64:
   3681 		case STORAGE_I64:
   3682 		case STORAGE_SIZE: // XXX: arch
   3683 		case STORAGE_UINTPTR:
   3684 			item->type = QD_VALUE;
   3685 			item->value = constl((uint64_t)constant->uval);
   3686 			break;
   3687 		default:
   3688 			assert(0);
   3689 		}
   3690 		break;
   3691 	case STORAGE_TUPLE:
   3692 		for (const struct tuple_constant *tuple = constant->tuple;
   3693 				tuple; tuple = tuple->next) {
   3694 			item = gen_data_item(ctx, tuple->value, item);
   3695 			if (tuple->next) {
   3696 				const struct type_tuple *f1 = tuple->field;
   3697 				const struct type_tuple *f2 = tuple->next->field;
   3698 				if (f2->offset != f1->offset + f1->type->size) {
   3699 					item->next = xcalloc(1,
   3700 						sizeof(struct qbe_data_item));
   3701 					item = item->next;
   3702 					item->type = QD_ZEROED;
   3703 					item->zeroed = f2->offset -
   3704 						(f1->offset + f1->type->size);
   3705 				}
   3706 
   3707 
   3708 				item->next = xcalloc(1,
   3709 					sizeof(struct qbe_data_item));
   3710 				item = item->next;
   3711 			} else {
   3712 				const struct type_tuple *fi = tuple->field;
   3713 				if (fi->offset + fi->type->size
   3714 						!= expr->result->size) {
   3715 					item->next = xcalloc(1,
   3716 						sizeof(struct qbe_data_item));
   3717 					item = item->next;
   3718 					item->type = QD_ZEROED;
   3719 					item->zeroed = expr->result->size
   3720 						- (fi->offset + fi->type->size);
   3721 				}
   3722 			}
   3723 		}
   3724 		break;
   3725 	case STORAGE_TAGGED:
   3726 		item->type = QD_VALUE;
   3727 		item->value = constw((uint32_t)constant->tagged.tag->id);
   3728 		if (type->align != builtin_type_uint.align) {
   3729 			item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3730 			item = item->next;
   3731 			item->type = QD_ZEROED;
   3732 			item->zeroed = type->align - builtin_type_uint.align;
   3733 		}
   3734 		if (constant->tagged.tag->size != 0) {
   3735 			item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3736 			item = item->next;
   3737 			item = gen_data_item(ctx, constant->tagged.value, item);
   3738 		}
   3739 		if (constant->tagged.tag->size < type->size - type->align) {
   3740 			item->next = xcalloc(1, sizeof(struct qbe_data_item));
   3741 			item = item->next;
   3742 			item->type = QD_ZEROED;
   3743 			item->zeroed = type->size - type->align - constant->tagged.tag->size;
   3744 		}
   3745 		break;
   3746 	case STORAGE_UNION:
   3747 	case STORAGE_ALIAS:
   3748 	case STORAGE_ERROR:
   3749 	case STORAGE_FCONST:
   3750 	case STORAGE_FUNCTION:
   3751 	case STORAGE_ICONST:
   3752 	case STORAGE_RCONST:
   3753 	case STORAGE_NULL:
   3754 	case STORAGE_VALIST:
   3755 	case STORAGE_VOID:
   3756 		assert(0); // Invariant
   3757 	}
   3758 
   3759 	assert(item->type != QD_VALUE || item->value.type);
   3760 	return item;
   3761 }
   3762 
   3763 static void
   3764 gen_global_decl(struct gen_context *ctx, const struct declaration *decl)
   3765 {
   3766 	assert(decl->type == DECL_GLOBAL);
   3767 	const struct global_decl *global = &decl->global;
   3768 	if (!global->value) {
   3769 		return; // Forward declaration
   3770 	}
   3771 	struct qbe_def *qdef = xcalloc(1, sizeof(struct qbe_def));
   3772 	qdef->kind = Q_DATA;
   3773 	qdef->data.align = ALIGN_UNDEFINED;
   3774 	qdef->data.threadlocal = global->threadlocal;
   3775 	qdef->exported = decl->exported;
   3776 	qdef->name = ident_to_sym(&decl->ident);
   3777 	gen_data_item(ctx, global->value, &qdef->data.items);
   3778 	qbe_append_def(ctx->out, qdef);
   3779 }
   3780 
   3781 static void
   3782 gen_decl(struct gen_context *ctx, const struct declaration *decl)
   3783 {
   3784 	switch (decl->type) {
   3785 	case DECL_FUNC:
   3786 		gen_function_decl(ctx, decl);
   3787 		break;
   3788 	case DECL_GLOBAL:
   3789 		gen_global_decl(ctx, decl);
   3790 		break;
   3791 	case DECL_TYPE:
   3792 	case DECL_CONST:
   3793 		break; // no-op
   3794 	}
   3795 }
   3796 
   3797 void
   3798 gen(const struct unit *unit, struct type_store *store, struct qbe_program *out)
   3799 {
   3800 	struct gen_context ctx = {
   3801 		.out = out,
   3802 		.store = store,
   3803 		.ns = unit->ns,
   3804 		.arch = {
   3805 			.ptr = &qbe_long,
   3806 			.sz = &qbe_long,
   3807 		},
   3808 	};
   3809 	ctx.out->next = &ctx.out->defs;
   3810 	const struct declarations *decls = unit->declarations;
   3811 	while (decls) {
   3812 		gen_decl(&ctx, decls->decl);
   3813 		decls = decls->next;
   3814 	}
   3815 }