Add VCL_Shutdown to wait for VCL references to vanish

Before shutting down stevedores, we should wait for VCL references to go
away to, indirectly, ensure that all worker threads potentially using
storage functions are done.

See https://gitlab.com/uplex/varnish/slash/-/issues/61

Author: nigoroll

bin/varnishd/cache/cache_main.c

@@ -385,6 +385,20 @@ cli_quit(int sig)
 	WRONG("It's time for the big quit");
 }
 
+/*=====================================================================
+ * XXX Generalize?
+ */
+
+static void
+bit_set(volatile uint8_t *p, unsigned no)
+{
+	uint8_t b;
+
+	p += (no >> 3);
+	b = (0x80 >> (no & 7));
+	*p |= b;
+}
+
 /*=====================================================================
  * Run the child process
  */
@@ -490,12 +504,15 @@ child_main(int sigmagic, size_t altstksz)
 
 	CLI_Run();
 
+	bit_set(cache_param->debug_bits, DBG_VCLREL);
+
 	if (shutdown_delay > 0)
 		VTIM_sleep(shutdown_delay);
 
 	VCA_Shutdown();
 	BAN_Shutdown();
 	EXP_Shutdown();
+	VCL_Shutdown();
 	STV_close();
 
 	printf("Child dies\n");

bin/varnishd/cache/cache_varnishd.h

@@ -493,6 +493,7 @@ struct vsb *VCL_Rel_CliCtx(struct vrt_ctx **);
 void VCL_Panic(struct vsb *, const char *nm, const struct vcl *);
 void VCL_Poll(void);
 void VCL_Init(void);
+void VCL_Shutdown(void);
 
 #define VCL_MET_MAC(l,u,t,b) \
     void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \

bin/varnishd/cache/cache_vcl.c

@@ -1180,3 +1180,25 @@ VCL_Init(void)
 	Lck_New(&vcl_mtx, lck_vcl);
 	VSL_Setup(&vsl_cli, NULL, 0);
 }
+
+void
+VCL_Shutdown(void)
+{
+	struct vcl *vcl;
+	unsigned c = 0;
+
+	while (1) {
+		Lck_Lock(&vcl_mtx);
+		VTAILQ_FOREACH(vcl, &vcl_head, list)
+			if (vcl->busy)
+				break;
+		Lck_Unlock(&vcl_mtx);
+		if (vcl == NULL)
+			return;
+		if (++c % 10 == 0) {
+			fprintf(stderr, "shutdown waiting for vcl %u\t%s\n",
+				vcl->busy, vcl->loaded_name);
+		}
+		usleep(100 * 1000);
+	}
+}