s390/ap: ap bus userspace notifications for some bus conditions

This patch adds notifications to userspace for two important
conditions of the ap bus:

I) Initial ap bus scan done. This indicates that the initial
   scan of all the ap devices (cards, queues) is complete and
   ap devices have been build up for all the hardware found.
   This condition is signaled with
   1) An ap bus change uevent send to userspace with an environment
      key/value pair "INITSCAN=done":
	# udevadm monitor -k -p
	...
	KERNEL[97.830919] change   /devices/ap (ap)
	ACTION=change
	DEVPATH=/devices/ap
	SUBSYSTEM=ap
	INITSCAN=done
	SEQNUM=10421
   2) A sysfs attribute /sys/bus/ap/scans which shows the
      number of completed ap bus scans done since bus init.
      So a value of 1 or greater signals that the initial
      ap bus scan is complete.
   Note: The initial ap bus scan complete condition is fulfilled
   and will be signaled even if there was no ap resource found.

II) APQN driver bindings complete. This indicates that all
    APQNs have been bound to an zcrypt or alternate device
    driver. Only with the help of an device driver an APQN
    can be used for crypto load. So the binding complete
    condition is the starting point for user space to be
    sure all crypto resources on the ap bus are available
    for use.
    This condition is signaled with
    1) An ap bus change uevent send to userspace with an environment
       key/value pair "BINDINGS=complete":
	 # udevadm monitor -k -p
	 ...
	 KERNEL[97.830975] change   /devices/ap (ap)
	 ACTION=change
	 DEVPATH=/devices/ap
	 SUBSYSTEM=ap
	 BINDINGS=complete
	 SEQNUM=10422
    2) A sysfs attribute /sys/bus/ap/bindings showing
	 "<nr of bound apqns>/<total nr of apqns> (complete)"
       when all available apqns have been bound to device drivers, or
	 "<nr of bound apqns>/<total nr of apqns>"
       when there are some apqns not bound to an device driver.
    Note: The binding complete condition is also fulfilled, when
    there are no apqns available to bind any device driver. In
    this case the binding complete will be signaled AFTER init
    scan is done.
    Note: This condition may arise multiple times when after
    initial scan modifications on the bindings take place. For
    example a manual unbind of an APQN switches the binding
    complete condition off. When at a later time the unbound APQNs
    are bound with an device driver the binding is (again) complete
    resulting in another uevent and marking the bindings sysfs
    attribute with '(complete)'.

There is also a new function to be used within the kernel:

  int ap_wait_init_apqn_bindings_complete(unsigned long timeout)

Interface to wait for the AP bus to have done one initial ap bus
scan and all detected APQNs have been bound to device drivers.
If these both conditions are not fulfilled, this function blocks
on a condition with wait_for_completion_interruptible_timeout().
If these both conditions are fulfilled (before the timeout hits)
the return value is 0. If the timeout (in jiffies) hits instead
-ETIME is returned. On failures negative return values are
returned to the caller. Please note that further unbind/bind
actions after initial binding complete is through do not cause this
function to block again.

Reviewed-by: Ingo Franzki <ifranzki@linux.ibm.com>
Signed-off-by: Harald Freudenberger <freude@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

2 files changed, 166 insertions, 10 deletions

diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index ef738b42a092..13bd6b27f00e 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -1,11 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
- * Copyright IBM Corp. 2006, 2012
+ * Copyright IBM Corp. 2006, 2020
  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
  *	      Martin Schwidefsky <schwidefsky@de.ibm.com>
  *	      Ralph Wuerthner <rwuerthn@de.ibm.com>
  *	      Felix Beck <felix.beck@de.ibm.com>
  *	      Holger Dengler <hd@linux.vnet.ibm.com>
+ *	      Harald Freudenberger <freude@linux.ibm.com>
  *
  * Adjunct processor bus.
  */
@@ -73,6 +74,12 @@ EXPORT_SYMBOL(ap_perms);
 DEFINE_MUTEX(ap_perms_mutex);
 EXPORT_SYMBOL(ap_perms_mutex);
 
+/* # of bus scans since init */
+static atomic64_t ap_scan_bus_count;
+
+/* completion for initial APQN bindings complete */
+static DECLARE_COMPLETION(ap_init_apqn_bindings_complete);
+
 static struct ap_config_info *ap_qci_info;
 
 /*
@@ -577,23 +584,125 @@ static int ap_bus_match(struct device *dev, struct device_driver *drv)
  */
 static int ap_uevent(struct device *dev, struct kobj_uevent_env *env)
 {
+	int rc;
 	struct ap_device *ap_dev = to_ap_dev(dev);
-	int retval = 0;
 
-	if (!ap_dev)
-		return -ENODEV;
+	/* Uevents from ap bus core don't need extensions to the env */
+	if (dev == ap_root_device)
+		return 0;
 
 	/* Set up DEV_TYPE environment variable. */
-	retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
-	if (retval)
-		return retval;
+	rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
+	if (rc)
+		return rc;
 
 	/* Add MODALIAS= */
-	retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
+	rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);
+	if (rc)
+		return rc;
+
+	return 0;
+}
+
+static void ap_send_init_scan_done_uevent(void)
+{
+	char *envp[] = { "INITSCAN=done", NULL };
+
+	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
+}
+
+static void ap_send_bindings_complete_uevent(void)
+{
+	char *envp[] = { "BINDINGS=complete", NULL };
+
+	kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp);
+}
+
+/*
+ * calc # of bound APQNs
+ */
+
+struct __ap_calc_ctrs {
+	unsigned int apqns;
+	unsigned int bound;
+};
+
+static int __ap_calc_helper(struct device *dev, void *arg)
+{
+	struct __ap_calc_ctrs *pctrs = (struct __ap_calc_ctrs *) arg;
+
+	if (is_queue_dev(dev)) {
+		pctrs->apqns++;
+		if ((to_ap_dev(dev))->drv)
+			pctrs->bound++;
+	}
+
+	return 0;
+}
+
+static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound)
+{
+	struct __ap_calc_ctrs ctrs;
+
+	memset(&ctrs, 0, sizeof(ctrs));
+	bus_for_each_dev(&ap_bus_type, NULL, (void *) &ctrs, __ap_calc_helper);
 
-	return retval;
+	*apqns = ctrs.apqns;
+	*bound = ctrs.bound;
 }
 
+/*
+ * After initial ap bus scan do check if all existing APQNs are
+ * bound to device drivers.
+ */
+static void ap_check_bindings_complete(void)
+{
+	unsigned int apqns, bound;
+
+	if (atomic64_read(&ap_scan_bus_count) >= 1) {
+		ap_calc_bound_apqns(&apqns, &bound);
+		if (bound == apqns) {
+			if (!completion_done(&ap_init_apqn_bindings_complete)) {
+				complete_all(&ap_init_apqn_bindings_complete);
+				AP_DBF(DBF_INFO, "%s complete\n", __func__);
+			}
+			ap_send_bindings_complete_uevent();
+		}
+	}
+}
+
+/*
+ * Interface to wait for the AP bus to have done one initial ap bus
+ * scan and all detected APQNs have been bound to device drivers.
+ * If these both conditions are not fulfilled, this function blocks
+ * on a condition with wait_for_completion_interruptible_timeout().
+ * If these both conditions are fulfilled (before the timeout hits)
+ * the return value is 0. If the timeout (in jiffies) hits instead
+ * -ETIME is returned. On failures negative return values are
+ * returned to the caller.
+ */
+int ap_wait_init_apqn_bindings_complete(unsigned long timeout)
+{
+	long l;
+
+	if (completion_done(&ap_init_apqn_bindings_complete))
+		return 0;
+
+	if (timeout)
+		l = wait_for_completion_interruptible_timeout(
+			&ap_init_apqn_bindings_complete, timeout);
+	else
+		l = wait_for_completion_interruptible(
+			&ap_init_apqn_bindings_complete);
+	if (l < 0)
+		return l == -ERESTARTSYS ? -EINTR : l;
+	else if (l == 0 && timeout)
+		return -ETIME;
+
+	return 0;
+}
+EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete);
+
 static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data)
 {
 	if (is_queue_dev(dev) &&
@@ -719,7 +828,8 @@ static int ap_device_probe(struct device *dev)
 			hash_del(&to_ap_queue(dev)->hnode);
 		spin_unlock_bh(&ap_queues_lock);
 		ap_dev->drv = NULL;
-	}
+	} else
+		ap_check_bindings_complete();
 
 out:
 	if (rc)
@@ -749,6 +859,7 @@ static int ap_device_remove(struct device *dev)
 	if (is_queue_dev(dev))
 		hash_del(&to_ap_queue(dev)->hnode);
 	spin_unlock_bh(&ap_queues_lock);
+	ap_dev->drv = NULL;
 
 	put_device(dev);
 
@@ -1166,6 +1277,30 @@ static ssize_t aqmask_store(struct bus_type *bus, const char *buf,
 
 static BUS_ATTR_RW(aqmask);
 
+static ssize_t scans_show(struct bus_type *bus, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%llu\n",
+			 atomic64_read(&ap_scan_bus_count));
+}
+
+static BUS_ATTR_RO(scans);
+
+static ssize_t bindings_show(struct bus_type *bus, char *buf)
+{
+	int rc;
+	unsigned int apqns, n;
+
+	ap_calc_bound_apqns(&apqns, &n);
+	if (atomic64_read(&ap_scan_bus_count) >= 1 && n == apqns)
+		rc = scnprintf(buf, PAGE_SIZE, "%u/%u (complete)\n", n, apqns);
+	else
+		rc = scnprintf(buf, PAGE_SIZE, "%u/%u\n", n, apqns);
+
+	return rc;
+}
+
+static BUS_ATTR_RO(bindings);
+
 static struct bus_attribute *const ap_bus_attrs[] = {
 	&bus_attr_ap_domain,
 	&bus_attr_ap_control_domain_mask,
@@ -1179,6 +1314,8 @@ static struct bus_attribute *const ap_bus_attrs[] = {
 	&bus_attr_ap_max_adapter_id,
 	&bus_attr_apmask,
 	&bus_attr_aqmask,
+	&bus_attr_scans,
+	&bus_attr_bindings,
 	NULL,
 };
 
@@ -1608,6 +1745,12 @@ static void ap_scan_bus(struct work_struct *unused)
 				    ap_domain_index);
 	}
 
+	if (atomic64_inc_return(&ap_scan_bus_count) == 1) {
+		AP_DBF(DBF_DEBUG, "%s init scan complete\n", __func__);
+		ap_send_init_scan_done_uevent();
+		ap_check_bindings_complete();
+	}
+
 	mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ);
 }
 
@@ -1705,6 +1848,7 @@ static int __init ap_module_init(void)
 	rc = PTR_ERR_OR_ZERO(ap_root_device);
 	if (rc)
 		goto out_bus;
+	ap_root_device->bus = &ap_bus_type;
 
 	/* Setup the AP bus rescan timer. */
 	timer_setup(&ap_config_timer, ap_config_timeout, 0);
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 5029b80132aa..472efd3a755c 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -350,4 +350,16 @@ int ap_parse_mask_str(const char *str,
 		      unsigned long *bitmap, int bits,
 		      struct mutex *lock);
 
+/*
+ * Interface to wait for the AP bus to have done one initial ap bus
+ * scan and all detected APQNs have been bound to device drivers.
+ * If these both conditions are not fulfilled, this function blocks
+ * on a condition with wait_for_completion_killable_timeout().
+ * If these both conditions are fulfilled (before the timeout hits)
+ * the return value is 0. If the timeout (in jiffies) hits instead
+ * -ETIME is returned. On failures negative return values are
+ * returned to the caller.
+ */
+int ap_wait_init_apqn_bindings_complete(unsigned long timeout);
+
 #endif /* _AP_BUS_H_ */