Trigger iOS Shutter via BLE on iOS 16

[florianbuerger]

We would like the Fjorden Grip also to trigger the native iOS camera's shutter button, via sending a “volume up” command—similar to selfie sticks, which however use classic Bluetooth instead of BLE. Our implementation worked flawlessly in iOS 15, but stopped working in iOS 16. There is no documentation as to why. I wonder if anyone has encountered this issue, or understand what is happening?

----

To control the shutter button of the native iOS camera app, we have to implement an HID keyboard. This is only so that you can pair the grip via Settings.app, and then trigger the camera shutter. We are sending a volume up command, which then triggers the shutter. This is how all selfie-sticks work (via classic Bluetooth).

The grip advertises itself not as a full keyboard, but as a consumer control device. 

The expected behavior is:

• Pair grip via Settings.app → Bluetooth (our app/SDK is never involved)
• Open Camera.app
• Press the shutter button on the grip
• The shutter button in Camera.app is triggered, similar to pressing a volume button on the phone

On iOS 15 (and iOS 16 during the beta cycle in the summer before it was released), this works perfectly. Since iOS 16 was released, nothing happens anymore when pressing the shutter button. Our HID report map:

static uint8_t report_map[] = {
 0x05, 0x0C, // (GLOBAL) USAGE_PAGE 0x000C Consumer Device Page
 0x09, 0x01, // (LOCAL) USAGE 0x000C0001 Consumer Control
 0xA1, 0x01, // (MAIN) COLLECTION 0x01 Application
 0x19, 0x00, // (LOCAL) USAGE_MINIMUM
 0x2A, 0x9C, 0x02, // (LOCAL) USAGE_MAXIMUM
 0x15, 0x00, // (GLOBAL) LOGICAL_MINIMUM
 0x26, 0x9C, 0x02, // (GLOBAL) LOGICAL_MAXIMUM 0x029C (668)
 0x95, 0x01, // (GLOBAL) REPORT_COUNT 0x01 (1) Number of fields
 0x75, 0x10, // (GLOBAL) REPORT_SIZE 0x10 (16) Number of bits per field
 0x81, 0x00, // (MAIN) INPUT
 0xC0 // (MAIN) END_COLLECTION Application
};

I tried to find other selfie-sticks that work via BLE, but couldn't find any. Has anyone run into this issue?

[Sigurd]

Hi,

Can you check and make sure that all connection parameters match the requirements from the Apple accessory design guidelines?
https://developer.apple.com/accessories/Accessory-Design-Guidelines.pdf

florianbuerger said:

I wonder if anyone has encountered this issue, or understand what is happening?

We have seen reports that there have been some changes in iOS v16 on how these guidelines are being enforced. See e.g. this case:  RE: Getting continuous disconnects with iOS 16 

[florianbuerger]

Hey Sigurd, thanks for getting back to me. I’ll double-check with our engineer, but the device works flawlessly with using our app—which then doesn’t rely on HID, we are using our services/characteristics. But there are no disconnects.

[Sigurd]

Ok, let me know once you know more. Should the connection parameters turn out to be inline with the guidelines, then the next step in debugging this would be to take a sniffer trace: https://www.nordicsemi.com/Products/Development-tools/nrf-sniffer-for-bluetooth-le

[florianbuerger]

Okay, I can confirm that we adhere to the suggested intervals. Our accessory connects fine, and never gets disconnected by iOS.

We experimented with the report descriptor, and found a version that works on iOS 16. But, the same version does not work on iOS 15. Instead of receiving a single volume down command, iOS 15 treats it as a continuous volume up command. It seems the key release event is never sent. 

I was unable to capture a sniffer trace, for some reason the Wireshark plugin didn't work. I tried on macOS first, with no luck. I installed Windows 10 on an Intel MacBook, the plugin worked once, but then it stopped. We do have logs from running our firmware on the Nordic Dev Board. I've attached the updated HID report map.

#include <hid.h>

#include <zephyr/types.h>
#include <stddef.h>
#include <string.h>
#include <errno.h>
#include <sys/byteorder.h>
#include <zephyr.h>

#include <bluetooth/bluetooth.h>
#include <bluetooth/hci.h>
#include <bluetooth/conn.h>
#include <bluetooth/uuid.h>
#include <bluetooth/gatt.h>

#include <logging/log.h>
LOG_MODULE_REGISTER(hid);

// BLE HID Descriptor Report ID
#define MEDIA_KEYS_ID 0x01

#define KEY_VOLUME_UP (0xe9)
#define KEY_VOLUME_DOWN (0xea)
#define KEY_RELEASE (0x00)

#define KEY_DELAY K_MSEC(20)

#define STACKSIZE 1024
#define PRIORITY 7

enum {
	HIDS_REMOTE_WAKE = BIT(0),
	HIDS_NORMALLY_CONNECTABLE = BIT(1),
};

struct hids_info {
	uint16_t version;
	uint8_t code;
	uint8_t flags;
} __packed;

struct hids_report {
	uint8_t id;
	uint8_t type;
} __packed;

static struct hids_info info = {
	.version = 0x0101,
	.code = 33,
	.flags = HIDS_NORMALLY_CONNECTABLE,
};

enum {
	HIDS_INPUT = 0x01,
	HIDS_OUTPUT = 0x02,
	HIDS_FEATURE = 0x03,
};

enum {
	HIDS_BOOT_PROTOCOL = 0x0,
	HIDS_REPORT_PROTOCOL = 0x1,
};

static struct hids_report input = {
	.id = 0x01,
	.type = HIDS_INPUT,
};

static uint8_t protocol = HIDS_REPORT_PROTOCOL;
static uint8_t send_input;
static uint8_t ctrl_point;

static uint8_t report_map[] = {
0x05, 0x0C,                     // Usage Page (Consumer)
        0x09, 0x01,                     // Usage (Consumer Control)
        0xA1, 0x01,                     // Collection (Application)
        0x85, MEDIA_KEYS_ID,            //     Report Id (1)
        0x15, 0x00,                     //     Logical minimum (0)
        0x25, 0x01,                     //     Logical maximum (1)
        0x75, 0x01,                     //     Report Size (1)
        0x95, 0x01,                     //     Report Count (1)

        0x09, KEY_VOLUME_DOWN,          //     Usage (Volume Down)
        0x81, 0x02,                     //     Input (Data,Value,Relative,Bit Field)
        0x09, KEY_VOLUME_UP,            //     Usage (Volume Up)
        0x81, 0x02,                     //     Input (Data,Value,Relative,Bit Field)

        0xC0                            // End Collection	
};


static ssize_t read_info(struct bt_conn *conn, const struct bt_gatt_attr *attr, 
	void *buf, uint16_t len, uint16_t offset)
{
	return bt_gatt_attr_read(conn, attr, buf, len, offset, attr->user_data, sizeof(struct hids_info));
}

static ssize_t read_report_map(struct bt_conn *conn, const struct bt_gatt_attr *attr, 
	void *buf, uint16_t len, uint16_t offset)
{
	return bt_gatt_attr_read(conn, attr, buf, len, offset, report_map, sizeof(report_map));
}

static ssize_t read_report(struct bt_conn *conn, const struct bt_gatt_attr *attr, 
	void *buf, uint16_t len, uint16_t offset)
{
	return bt_gatt_attr_read(conn, attr, buf, len, offset, attr->user_data, sizeof(struct hids_report));
}

static ssize_t read_protocol(struct bt_conn *conn, const struct bt_gatt_attr *attr, 
	void *buf, uint16_t len, uint16_t offset)
{
	return bt_gatt_attr_read(conn, attr, buf, len, offset, attr->user_data, sizeof(protocol));
}

static void input_ccc_changed(const struct bt_gatt_attr *attr, uint16_t value)
{
	send_input = (value == BT_GATT_CCC_NOTIFY) ? 1 : 0;
}

static ssize_t read_input_report(struct bt_conn *conn, 
	const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset)
{
	return bt_gatt_attr_read(conn, attr, buf, len, offset, NULL, 0);
}

static ssize_t write_ctrl_point(struct bt_conn *conn, 
	const struct bt_gatt_attr *attr, const void *buf, uint16_t len, 
	uint16_t offset, uint8_t flags)
{
	uint8_t *value = attr->user_data;

	if (offset + len > sizeof(ctrl_point)) 
    {
		return BT_GATT_ERR(BT_ATT_ERR_INVALID_OFFSET);
	}

	memcpy(value + offset, buf, len);

	return len;
}

/* HID Service Declaration */
BT_GATT_SERVICE_DEFINE(hid_service,
	BT_GATT_PRIMARY_SERVICE(BT_UUID_HIDS),
	BT_GATT_CHARACTERISTIC(BT_UUID_HIDS_INFO, 
		BT_GATT_CHRC_READ, BT_GATT_PERM_READ, read_info, NULL, &info),
	BT_GATT_CHARACTERISTIC(BT_UUID_HIDS_REPORT_MAP, 
		BT_GATT_CHRC_READ, BT_GATT_PERM_READ, read_report_map, NULL, NULL),
	BT_GATT_CHARACTERISTIC(BT_UUID_HIDS_REPORT, 
		BT_GATT_CHRC_READ | BT_GATT_CHRC_NOTIFY, BT_GATT_PERM_READ_ENCRYPT, 
		read_input_report, NULL, NULL),
	BT_GATT_CCC(input_ccc_changed, 
		BT_GATT_PERM_READ_ENCRYPT | BT_GATT_PERM_WRITE_ENCRYPT),
	BT_GATT_DESCRIPTOR(BT_UUID_HIDS_REPORT_REF, 
		BT_GATT_PERM_READ, read_report, NULL, &input),
    BT_GATT_CHARACTERISTIC(BT_UUID_HIDS_PROTOCOL_MODE, 
		BT_GATT_CHRC_READ, BT_GATT_PERM_READ, read_protocol, NULL, &protocol),
	BT_GATT_CHARACTERISTIC(BT_UUID_HIDS_CTRL_POINT, 
		BT_GATT_CHRC_WRITE_WITHOUT_RESP, BT_GATT_PERM_WRITE, NULL, 
		write_ctrl_point, &ctrl_point),
);

struct key_report 
{
	int reportId;
    int keycode;
};

K_MSGQ_DEFINE(key_report_queue, sizeof(struct key_report), 8, 4);

static void 
send_key(int reportId, int keycode)
{
	struct key_report report = { reportId, keycode };

	LOG_INF("Sending key report to HID thread {%x, %x}", report.reportId, report.keycode);
	
	int rc = k_msgq_put(&key_report_queue, &report, K_NO_WAIT);

	if (rc != 0)
	{
		LOG_ERR("Failed to send key report");
	}
}

void 
hid_thread_handler(void)
{
	struct key_report report = { 0 };

	for (;;)
	{
		k_sleep(KEY_DELAY);

		k_msgq_get(&key_report_queue, &report, K_FOREVER);

		LOG_INF("[HID Thread] Received key report {%x, %x}", report.reportId, report.keycode);

		uint8_t raw_cc_report[] = { report.reportId, report.keycode };
		LOG_HEXDUMP_INF(raw_cc_report, sizeof(raw_cc_report), "Hex dump");
		bt_gatt_notify(NULL, hid_service.attrs + 5, raw_cc_report, sizeof(raw_cc_report));
	}
}

K_THREAD_DEFINE(hid_thread, STACKSIZE, hid_thread_handler, NULL, NULL, NULL, PRIORITY, 0, 0);

void
hid_send_press_event(void)
{
    LOG_INF("Sending HID press event");

	if (send_input == 0)
	{
		LOG_INF("iOS is not listening for the events");
		return;
	}

    send_key(MEDIA_KEYS_ID, KEY_VOLUME_DOWN);
}

void
hid_send_release_event(void)
{
	LOG_INF("Sending HID release event");

	if (send_input == 0)
	{
		LOG_INF("iOS is not listening for the events");
		return;
	}

    send_key(0, KEY_RELEASE);
}

[Sigurd]

Hi,

florianbuerger said:

I've attached the updated HID report map.

Could you try to set MEDIA_KEYS_ID to 0 ?

More context for this suggestion can be found in this PR: https://github.com/nrfconnect/sdk-nrf/pull/8772