From d083ba906b587a5e37d10bbf0be81309a71ef074 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 18:19:12 +0200 Subject: [PATCH 1/6] asus Signed-off-by: Peter Jung --- .../ABI/testing/sysfs-platform-asus-wmi | 17 + drivers/hid/Kconfig | 10 + drivers/hid/Makefile | 1 + drivers/hid/hid-asus-ally.c | 2197 +++++++++++++++++ drivers/hid/hid-asus-ally.h | 398 +++ drivers/hid/hid-asus.c | 415 +++- drivers/hid/hid-asus.h | 13 + drivers/hid/hid-ids.h | 3 +- drivers/platform/x86/Kconfig | 23 + drivers/platform/x86/Makefile | 1 + drivers/platform/x86/asus-armoury.c | 1174 +++++++++ drivers/platform/x86/asus-armoury.h | 1278 ++++++++++ drivers/platform/x86/asus-nb-wmi.c | 30 +- drivers/platform/x86/asus-wmi.c | 328 ++- drivers/platform/x86/asus-wmi.h | 3 +- include/linux/platform_data/x86/asus-wmi.h | 51 + 16 files changed, 5765 insertions(+), 177 deletions(-) create mode 100644 drivers/hid/hid-asus-ally.c create mode 100644 drivers/hid/hid-asus-ally.h create mode 100644 drivers/hid/hid-asus.h create mode 100644 drivers/platform/x86/asus-armoury.c create mode 100644 drivers/platform/x86/asus-armoury.h diff --git a/Documentation/ABI/testing/sysfs-platform-asus-wmi b/Documentation/ABI/testing/sysfs-platform-asus-wmi index 28144371a0f1..765d50b0d9df 100644 --- a/Documentation/ABI/testing/sysfs-platform-asus-wmi +++ b/Documentation/ABI/testing/sysfs-platform-asus-wmi @@ -63,6 +63,7 @@ Date: Aug 2022 KernelVersion: 6.1 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Switch the GPU hardware MUX mode. Laptops with this feature can can be toggled to boot with only the dGPU (discrete mode) or in standard Optimus/Hybrid mode. On switch a reboot is required: @@ -75,6 +76,7 @@ Date: Aug 2022 KernelVersion: 5.17 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Disable discrete GPU: * 0 - Enable dGPU, * 1 - Disable dGPU @@ -84,6 +86,7 @@ Date: Aug 2022 KernelVersion: 5.17 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Enable the external GPU paired with ROG X-Flow laptops. Toggling this setting will also trigger ACPI to disable the dGPU: @@ -95,6 +98,7 @@ Date: Aug 2022 KernelVersion: 5.17 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Enable an LCD response-time boost to reduce or remove ghosting: * 0 - Disable, * 1 - Enable @@ -104,6 +108,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Get the current charging mode being used: * 1 - Barrel connected charger, * 2 - USB-C charging @@ -114,6 +119,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Show if the egpu (XG Mobile) is correctly connected: * 0 - False, * 1 - True @@ -123,6 +129,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Change the mini-LED mode: * 0 - Single-zone, * 1 - Multi-zone @@ -133,6 +140,7 @@ Date: Apr 2024 KernelVersion: 6.10 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON List the available mini-led modes. What: /sys/devices/platform//ppt_pl1_spl @@ -140,6 +148,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the Package Power Target total of CPU: PL1 on Intel, SPL on AMD. Shown on Intel+Nvidia or AMD+Nvidia based systems: @@ -150,6 +159,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the Slow Package Power Tracking Limit of CPU: PL2 on Intel, SPPT, on AMD. Shown on Intel+Nvidia or AMD+Nvidia based systems: @@ -160,6 +170,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the Fast Package Power Tracking Limit of CPU. AMD+Nvidia only: * min=5, max=250 @@ -168,6 +179,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the APU SPPT limit. Shown on full AMD systems only: * min=5, max=130 @@ -176,6 +188,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the platform SPPT limit. Shown on full AMD systems only: * min=5, max=130 @@ -184,6 +197,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the dynamic boost limit of the Nvidia dGPU: * min=5, max=25 @@ -192,6 +206,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the target temperature limit of the Nvidia dGPU: * min=75, max=87 @@ -200,6 +215,7 @@ Date: Apr 2024 KernelVersion: 6.10 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set if the BIOS POST sound is played on boot. * 0 - False, * 1 - True @@ -209,6 +225,7 @@ Date: Apr 2024 KernelVersion: 6.10 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set if the MCU can go in to low-power mode on system sleep * 0 - False, * 1 - True diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 43859fc75747..5bc1f086b855 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -178,6 +178,7 @@ config HID_APPLETB_KBD config HID_ASUS tristate "Asus" depends on USB_HID + depends on LEDS_CLASS_MULTICOLOR depends on LEDS_CLASS depends on ASUS_WMI || ASUS_WMI=n select POWER_SUPPLY @@ -191,6 +192,15 @@ config HID_ASUS - GL553V series - GL753V series +config HID_ASUS_ALLY + tristate "Asus Ally gamepad configuration support" + depends on USB_HID + depends on LEDS_CLASS + depends on LEDS_CLASS_MULTICOLOR + select POWER_SUPPLY + help + Support for configuring the Asus ROG Ally gamepad using attributes. + config HID_AUREAL tristate "Aureal" help diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 10ae5dedbd84..958f67193c85 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -33,6 +33,7 @@ obj-$(CONFIG_HID_APPLETB_BL) += hid-appletb-bl.o obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o obj-$(CONFIG_HID_CREATIVE_SB0540) += hid-creative-sb0540.o obj-$(CONFIG_HID_ASUS) += hid-asus.o +obj-$(CONFIG_HID_ASUS_ALLY) += hid-asus-ally.o obj-$(CONFIG_HID_AUREAL) += hid-aureal.o obj-$(CONFIG_HID_BELKIN) += hid-belkin.o obj-$(CONFIG_HID_BETOP_FF) += hid-betopff.o diff --git a/drivers/hid/hid-asus-ally.c b/drivers/hid/hid-asus-ally.c new file mode 100644 index 000000000000..e78625f70c44 --- /dev/null +++ b/drivers/hid/hid-asus-ally.c @@ -0,0 +1,2197 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2023 Luke Jones + */ + +#include "linux/compiler_attributes.h" +#include "linux/device.h" +#include +#include +#include "linux/pm.h" +#include "linux/printk.h" +#include "linux/slab.h" +#include +#include +#include +#include +#include + +#include "hid-ids.h" +#include "hid-asus.h" +#include "hid-asus-ally.h" + +#define DEBUG + +#define READY_MAX_TRIES 3 +#define FEATURE_REPORT_ID 0x0d +#define FEATURE_ROG_ALLY_REPORT_ID 0x5a +#define FEATURE_ROG_ALLY_CODE_PAGE 0xD1 +#define FEATURE_ROG_ALLY_REPORT_SIZE 64 +#define ALLY_X_INPUT_REPORT_USB 0x0B +#define ALLY_X_INPUT_REPORT_USB_SIZE 16 + +#define ROG_ALLY_REPORT_SIZE 64 +#define ROG_ALLY_X_MIN_MCU 313 +#define ROG_ALLY_MIN_MCU 319 + +#define FEATURE_KBD_LED_REPORT_ID1 0x5d +#define FEATURE_KBD_LED_REPORT_ID2 0x5e + +#define BTN_DATA_LEN 11; +#define BTN_CODE_BYTES_LEN 8 + +static const u8 EC_INIT_STRING[] = { 0x5A, 'A', 'S', 'U', 'S', ' ', 'T', 'e','c', 'h', '.', 'I', 'n', 'c', '.', '\0' }; +static const u8 EC_MODE_LED_APPLY[] = { 0x5A, 0xB4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +static const u8 EC_MODE_LED_SET[] = { 0x5A, 0xB5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +static const u8 FORCE_FEEDBACK_OFF[] = { 0x0D, 0x0F, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x00, 0xEB }; + +static const struct hid_device_id rog_ally_devices[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X) }, + {} +}; + +struct btn_code_map { + u64 code; + const char *name; +}; + +static const struct btn_code_map ally_btn_codes[] = { + { 0, "NONE" }, + /* Gamepad button codes */ + { BTN_PAD_A, "PAD_A" }, + { BTN_PAD_B, "PAD_B" }, + { BTN_PAD_X, "PAD_X" }, + { BTN_PAD_Y, "PAD_Y" }, + { BTN_PAD_LB, "PAD_LB" }, + { BTN_PAD_RB, "PAD_RB" }, + { BTN_PAD_LS, "PAD_LS" }, + { BTN_PAD_RS, "PAD_RS" }, + { BTN_PAD_DPAD_UP, "PAD_DPAD_UP" }, + { BTN_PAD_DPAD_DOWN, "PAD_DPAD_DOWN" }, + { BTN_PAD_DPAD_LEFT, "PAD_DPAD_LEFT" }, + { BTN_PAD_DPAD_RIGHT, "PAD_DPAD_RIGHT" }, + { BTN_PAD_VIEW, "PAD_VIEW" }, + { BTN_PAD_MENU, "PAD_MENU" }, + { BTN_PAD_XBOX, "PAD_XBOX" }, + + /* Triggers mapped to keyboard codes */ + { BTN_KB_M2, "KB_M2" }, + { BTN_KB_M1, "KB_M1" }, + { BTN_KB_ESC, "KB_ESC" }, + { BTN_KB_F1, "KB_F1" }, + { BTN_KB_F2, "KB_F2" }, + { BTN_KB_F3, "KB_F3" }, + { BTN_KB_F4, "KB_F4" }, + { BTN_KB_F5, "KB_F5" }, + { BTN_KB_F6, "KB_F6" }, + { BTN_KB_F7, "KB_F7" }, + { BTN_KB_F8, "KB_F8" }, + { BTN_KB_F9, "KB_F9" }, + { BTN_KB_F10, "KB_F10" }, + { BTN_KB_F11, "KB_F11" }, + { BTN_KB_F12, "KB_F12" }, + { BTN_KB_F14, "KB_F14" }, + { BTN_KB_F15, "KB_F15" }, + { BTN_KB_BACKTICK, "KB_BACKTICK" }, + { BTN_KB_1, "KB_1" }, + { BTN_KB_2, "KB_2" }, + { BTN_KB_3, "KB_3" }, + { BTN_KB_4, "KB_4" }, + { BTN_KB_5, "KB_5" }, + { BTN_KB_6, "KB_6" }, + { BTN_KB_7, "KB_7" }, + { BTN_KB_8, "KB_8" }, + { BTN_KB_9, "KB_9" }, + { BTN_KB_0, "KB_0" }, + { BTN_KB_HYPHEN, "KB_HYPHEN" }, + { BTN_KB_EQUALS, "KB_EQUALS" }, + { BTN_KB_BACKSPACE, "KB_BACKSPACE" }, + { BTN_KB_TAB, "KB_TAB" }, + { BTN_KB_Q, "KB_Q" }, + { BTN_KB_W, "KB_W" }, + { BTN_KB_E, "KB_E" }, + { BTN_KB_R, "KB_R" }, + { BTN_KB_T, "KB_T" }, + { BTN_KB_Y, "KB_Y" }, + { BTN_KB_U, "KB_U" }, + { BTN_KB_O, "KB_O" }, + { BTN_KB_P, "KB_P" }, + { BTN_KB_LBRACKET, "KB_LBRACKET" }, + { BTN_KB_RBRACKET, "KB_RBRACKET" }, + { BTN_KB_BACKSLASH, "KB_BACKSLASH" }, + { BTN_KB_CAPS, "KB_CAPS" }, + { BTN_KB_A, "KB_A" }, + { BTN_KB_S, "KB_S" }, + { BTN_KB_D, "KB_D" }, + { BTN_KB_F, "KB_F" }, + { BTN_KB_G, "KB_G" }, + { BTN_KB_H, "KB_H" }, + { BTN_KB_J, "KB_J" }, + { BTN_KB_K, "KB_K" }, + { BTN_KB_L, "KB_L" }, + { BTN_KB_SEMI, "KB_SEMI" }, + { BTN_KB_QUOTE, "KB_QUOTE" }, + { BTN_KB_RET, "KB_RET" }, + { BTN_KB_LSHIFT, "KB_LSHIFT" }, + { BTN_KB_Z, "KB_Z" }, + { BTN_KB_X, "KB_X" }, + { BTN_KB_C, "KB_C" }, + { BTN_KB_V, "KB_V" }, + { BTN_KB_B, "KB_B" }, + { BTN_KB_N, "KB_N" }, + { BTN_KB_M, "KB_M" }, + { BTN_KB_COMMA, "KB_COMMA" }, + { BTN_KB_PERIOD, "KB_PERIOD" }, + { BTN_KB_RSHIFT, "KB_RSHIFT" }, + { BTN_KB_LCTL, "KB_LCTL" }, + { BTN_KB_META, "KB_META" }, + { BTN_KB_LALT, "KB_LALT" }, + { BTN_KB_SPACE, "KB_SPACE" }, + { BTN_KB_RALT, "KB_RALT" }, + { BTN_KB_MENU, "KB_MENU" }, + { BTN_KB_RCTL, "KB_RCTL" }, + { BTN_KB_PRNTSCN, "KB_PRNTSCN" }, + { BTN_KB_SCRLCK, "KB_SCRLCK" }, + { BTN_KB_PAUSE, "KB_PAUSE" }, + { BTN_KB_INS, "KB_INS" }, + { BTN_KB_HOME, "KB_HOME" }, + { BTN_KB_PGUP, "KB_PGUP" }, + { BTN_KB_DEL, "KB_DEL" }, + { BTN_KB_END, "KB_END" }, + { BTN_KB_PGDWN, "KB_PGDWN" }, + { BTN_KB_UP_ARROW, "KB_UP_ARROW" }, + { BTN_KB_DOWN_ARROW, "KB_DOWN_ARROW" }, + { BTN_KB_LEFT_ARROW, "KB_LEFT_ARROW" }, + { BTN_KB_RIGHT_ARROW, "KB_RIGHT_ARROW" }, + + /* Numpad mappings */ + { BTN_NUMPAD_LOCK, "NUMPAD_LOCK" }, + { BTN_NUMPAD_FWDSLASH, "NUMPAD_FWDSLASH" }, + { BTN_NUMPAD_ASTERISK, "NUMPAD_ASTERISK" }, + { BTN_NUMPAD_HYPHEN, "NUMPAD_HYPHEN" }, + { BTN_NUMPAD_0, "NUMPAD_0" }, + { BTN_NUMPAD_1, "NUMPAD_1" }, + { BTN_NUMPAD_2, "NUMPAD_2" }, + { BTN_NUMPAD_3, "NUMPAD_3" }, + { BTN_NUMPAD_4, "NUMPAD_4" }, + { BTN_NUMPAD_5, "NUMPAD_5" }, + { BTN_NUMPAD_6, "NUMPAD_6" }, + { BTN_NUMPAD_7, "NUMPAD_7" }, + { BTN_NUMPAD_8, "NUMPAD_8" }, + { BTN_NUMPAD_9, "NUMPAD_9" }, + { BTN_NUMPAD_PLUS, "NUMPAD_PLUS" }, + { BTN_NUMPAD_ENTER, "NUMPAD_ENTER" }, + { BTN_NUMPAD_PERIOD, "NUMPAD_PERIOD" }, + + /* Mouse mappings */ + { BTN_MOUSE_LCLICK, "MOUSE_LCLICK" }, + { BTN_MOUSE_RCLICK, "MOUSE_RCLICK" }, + { BTN_MOUSE_MCLICK, "MOUSE_MCLICK" }, + { BTN_MOUSE_WHEEL_UP, "MOUSE_WHEEL_UP" }, + { BTN_MOUSE_WHEEL_DOWN, "MOUSE_WHEEL_DOWN" }, + + /* Media mappings */ + { BTN_MEDIA_SCREENSHOT, "MEDIA_SCREENSHOT" }, + { BTN_MEDIA_SHOW_KEYBOARD, "MEDIA_SHOW_KEYBOARD" }, + { BTN_MEDIA_SHOW_DESKTOP, "MEDIA_SHOW_DESKTOP" }, + { BTN_MEDIA_START_RECORDING, "MEDIA_START_RECORDING" }, + { BTN_MEDIA_MIC_OFF, "MEDIA_MIC_OFF" }, + { BTN_MEDIA_VOL_DOWN, "MEDIA_VOL_DOWN" }, + { BTN_MEDIA_VOL_UP, "MEDIA_VOL_UP" }, +}; +static const size_t keymap_len = ARRAY_SIZE(ally_btn_codes); + +/* byte_array must be >= 8 in length */ +static void btn_code_to_byte_array(u64 keycode, u8 *byte_array) +{ + /* Convert the u64 to bytes[8] */ + for (int i = 0; i < 8; ++i) { + byte_array[i] = (keycode >> (56 - 8 * i)) & 0xFF; + } +} + +static u64 name_to_btn(const char *name) +{ + int len = strcspn(name, "\n"); + for (size_t i = 0; i < keymap_len; ++i) { + if (strncmp(ally_btn_codes[i].name, name, len) == 0) { + return ally_btn_codes[i].code; + } + } + return -EINVAL; +} + +static const char* btn_to_name(u64 key) +{ + for (size_t i = 0; i < keymap_len; ++i) { + if (ally_btn_codes[i].code == key) { + return ally_btn_codes[i].name; + } + } + return NULL; +} + +struct btn_data { + u64 button; + u64 macro; + bool turbo; +}; + +struct btn_mapping { + struct btn_data btn_a; + struct btn_data btn_b; + struct btn_data btn_x; + struct btn_data btn_y; + struct btn_data btn_lb; + struct btn_data btn_rb; + struct btn_data btn_ls; + struct btn_data btn_rs; + struct btn_data btn_lt; + struct btn_data btn_rt; + struct btn_data dpad_up; + struct btn_data dpad_down; + struct btn_data dpad_left; + struct btn_data dpad_right; + struct btn_data btn_view; + struct btn_data btn_menu; + struct btn_data btn_m1; + struct btn_data btn_m2; +}; + +struct deadzone { + u8 inner; + u8 outer; +}; + +struct response_curve { + uint8_t move_pct_1; + uint8_t response_pct_1; + uint8_t move_pct_2; + uint8_t response_pct_2; + uint8_t move_pct_3; + uint8_t response_pct_3; + uint8_t move_pct_4; + uint8_t response_pct_4; +} __packed; + +struct js_axis_calibrations { + uint16_t left_y_stable; + uint16_t left_y_min; + uint16_t left_y_max; + uint16_t left_x_stable; + uint16_t left_x_min; + uint16_t left_x_max; + uint16_t right_y_stable; + uint16_t right_y_min; + uint16_t right_y_max; + uint16_t right_x_stable; + uint16_t right_x_min; + uint16_t right_x_max; +} __packed; + +struct tr_axis_calibrations { + uint16_t left_stable; + uint16_t left_max; + uint16_t right_stable; + uint16_t right_max; +} __packed; + +/* ROG Ally has many settings related to the gamepad, all using the same n-key endpoint */ +struct ally_gamepad_cfg { + struct hid_device *hdev; + struct input_dev *input; + + enum xpad_mode mode; + /* + * index: [mode] + */ + struct btn_mapping key_mapping[xpad_mode_mouse]; + /* + * index: left, right + * max: 64 + */ + u8 vibration_intensity[2]; + + /* deadzones */ + struct deadzone ls_dz; // left stick + struct deadzone rs_dz; // right stick + struct deadzone lt_dz; // left trigger + struct deadzone rt_dz; // right trigger + /* anti-deadzones */ + u8 ls_adz; // left stick + u8 rs_adz; // right stick + /* joystick response curves */ + struct response_curve ls_rc; + struct response_curve rs_rc; + + struct js_axis_calibrations js_cal; + struct tr_axis_calibrations tr_cal; +}; + +/* The hatswitch outputs integers, we use them to index this X|Y pair */ +static const int hat_values[][2] = { + { 0, 0 }, { 0, -1 }, { 1, -1 }, { 1, 0 }, { 1, 1 }, + { 0, 1 }, { -1, 1 }, { -1, 0 }, { -1, -1 }, +}; + +/* rumble packet structure */ +struct ff_data { + u8 enable; + u8 magnitude_left; + u8 magnitude_right; + u8 magnitude_strong; + u8 magnitude_weak; + u8 pulse_sustain_10ms; + u8 pulse_release_10ms; + u8 loop_count; +} __packed; + +struct ff_report { + u8 report_id; + struct ff_data ff; +} __packed; + +struct ally_x_input_report { + uint16_t x, y; + uint16_t rx, ry; + uint16_t z, rz; + uint8_t buttons[4]; +} __packed; + +struct ally_x_device { + struct input_dev *input; + struct hid_device *hdev; + spinlock_t lock; + + struct ff_report *ff_packet; + struct work_struct output_worker; + bool output_worker_initialized; + /* Prevent multiple queued event due to the enforced delay in worker */ + bool update_qam_btn; + /* Set if the QAM and AC buttons emit Xbox and Xbox+A */ + bool qam_btns_steam_mode; + bool update_ff; +}; + +struct ally_rgb_dev { + struct hid_device *hdev; + struct led_classdev_mc led_rgb_dev; + struct work_struct work; + bool output_worker_initialized; + spinlock_t lock; + + bool removed; + bool update_rgb; + uint8_t red[4]; + uint8_t green[4]; + uint8_t blue[4]; +}; + +struct ally_rgb_data { + uint8_t brightness; + uint8_t red[4]; + uint8_t green[4]; + uint8_t blue[4]; + bool initialized; +}; + +static struct ally_drvdata { + struct hid_device *hdev; + struct ally_x_device *ally_x; + struct ally_gamepad_cfg *gamepad_cfg; + struct ally_rgb_dev *led_rgb_dev; + struct ally_rgb_data led_rgb_data; + uint mcu_version; +} drvdata; + +static void reverse_bytes_in_pairs(u8 *buf, size_t size) { + uint16_t *word_ptr; + size_t i; + + for (i = 0; i < size; i += 2) { + if (i + 1 < size) { + word_ptr = (uint16_t *)&buf[i]; + *word_ptr = cpu_to_be16(*word_ptr); + } + } +} + +/** + * asus_dev_set_report - send set report request to device. + * + * @hdev: hid device + * @buf: in/out data to transfer + * @len: length of buf + * + * Return: count of data transferred, negative if error + * + * Same behavior as hid_hw_raw_request. Note that the input buffer is duplicated. + */ +static int asus_dev_set_report(struct hid_device *hdev, const u8 *buf, size_t len) +{ + unsigned char *dmabuf; + int ret; + + dmabuf = kmemdup(buf, len, GFP_KERNEL); + if (!dmabuf) + return -ENOMEM; + + ret = hid_hw_raw_request(hdev, buf[0], dmabuf, len, HID_FEATURE_REPORT, + HID_REQ_SET_REPORT); + kfree(dmabuf); + + return ret; +} + +/** + * asus_dev_get_report - send get report request to device. + * + * @hdev: hid device + * @out: buffer to write output data in to + * @len: length the output buffer provided + * + * Return: count of data transferred, negative if error + * + * Same behavior as hid_hw_raw_request. + */ +static int asus_dev_get_report(struct hid_device *hdev, u8 *out, size_t len) +{ + return hid_hw_raw_request(hdev, FEATURE_REPORT_ID, out, len, + HID_FEATURE_REPORT, HID_REQ_GET_REPORT); +} + +static u8 get_endpoint_address(struct hid_device *hdev) +{ + struct usb_interface *intf; + struct usb_host_endpoint *ep; + + intf = to_usb_interface(hdev->dev.parent); + + if (intf) { + ep = intf->cur_altsetting->endpoint; + if (ep) { + return ep->desc.bEndpointAddress; + } + } + + return -ENODEV; +} + +/**************************************************************************************************/ +/* ROG Ally gamepad configuration */ +/**************************************************************************************************/ + +/* This should be called before any attempts to set device functions */ +static int ally_gamepad_check_ready(struct hid_device *hdev) +{ + int ret, count; + u8 *hidbuf; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + ret = 0; + for (count = 0; count < READY_MAX_TRIES; count++) { + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + hidbuf[2] = xpad_cmd_check_ready; + hidbuf[3] = 01; + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + hid_dbg(hdev, "ROG Ally check failed set report: %d\n", ret); + + hidbuf[0] = hidbuf[1] = hidbuf[2] = hidbuf[3] = 0; + ret = asus_dev_get_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + hid_dbg(hdev, "ROG Ally check failed get report: %d\n", ret); + + ret = hidbuf[2] == xpad_cmd_check_ready; + if (ret) + break; + usleep_range( + 1000, + 2000); /* don't spam the entire loop in less than USB response time */ + } + + if (count == READY_MAX_TRIES) + hid_warn(hdev, "ROG Ally never responded with a ready\n"); + + kfree(hidbuf); + return ret; +} + +/* VIBRATION INTENSITY ****************************************************************************/ +static ssize_t gamepad_vibration_intensity_index_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "left right\n"); +} + +ALLY_DEVICE_ATTR_RO(gamepad_vibration_intensity_index, vibration_intensity_index); + +static ssize_t _gamepad_apply_intensity(struct hid_device *hdev, + struct ally_gamepad_cfg *ally_cfg) +{ + u8 *hidbuf; + int ret; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + hidbuf[2] = xpad_cmd_set_vibe_intensity; + hidbuf[3] = xpad_cmd_len_vibe_intensity; + hidbuf[4] = ally_cfg->vibration_intensity[0]; + hidbuf[5] = ally_cfg->vibration_intensity[1]; + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + goto report_fail; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + goto report_fail; + +report_fail: + kfree(hidbuf); + return ret; +} + +static ssize_t gamepad_vibration_intensity_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + return sysfs_emit( + buf, "%d %d\n", + ally_cfg->vibration_intensity[0], + ally_cfg->vibration_intensity[1]); +} + +static ssize_t gamepad_vibration_intensity_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + u32 left, right; + int ret; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + if (sscanf(buf, "%d %d", &left, &right) != 2) + return -EINVAL; + + if (left > 64 || right > 64) + return -EINVAL; + + ally_cfg->vibration_intensity[0] = left; + ally_cfg->vibration_intensity[1] = right; + + ret = _gamepad_apply_intensity(hdev, ally_cfg); + if (ret < 0) + return ret; + + return count; +} + +ALLY_DEVICE_ATTR_RW(gamepad_vibration_intensity, vibration_intensity); + +/* ANALOGUE DEADZONES *****************************************************************************/ +static ssize_t _gamepad_apply_deadzones(struct hid_device *hdev, + struct ally_gamepad_cfg *ally_cfg) +{ + u8 *hidbuf; + int ret; + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + return ret; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + hidbuf[2] = xpad_cmd_set_js_dz; + hidbuf[3] = xpad_cmd_len_deadzone; + hidbuf[4] = ally_cfg->ls_dz.inner; + hidbuf[5] = ally_cfg->ls_dz.outer; + hidbuf[6] = ally_cfg->rs_dz.inner; + hidbuf[7] = ally_cfg->rs_dz.outer; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + goto end; + + hidbuf[2] = xpad_cmd_set_tr_dz; + hidbuf[4] = ally_cfg->lt_dz.inner; + hidbuf[5] = ally_cfg->lt_dz.outer; + hidbuf[6] = ally_cfg->rt_dz.inner; + hidbuf[7] = ally_cfg->rt_dz.outer; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + goto end; + +end: + kfree(hidbuf); + return ret; +} + +static void _gamepad_set_deadzones_default(struct ally_gamepad_cfg *ally_cfg) +{ + ally_cfg->ls_dz.inner = 0x00; + ally_cfg->ls_dz.outer = 0x64; + ally_cfg->rs_dz.inner = 0x00; + ally_cfg->rs_dz.outer = 0x64; + ally_cfg->lt_dz.inner = 0x00; + ally_cfg->lt_dz.outer = 0x64; + ally_cfg->rt_dz.inner = 0x00; + ally_cfg->rt_dz.outer = 0x64; +} + +static ssize_t axis_xyz_deadzone_index_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "inner outer\n"); +} + +ALLY_DEVICE_ATTR_RO(axis_xyz_deadzone_index, deadzone_index); + +ALLY_DEADZONES(axis_xy_left, ls_dz); +ALLY_DEADZONES(axis_xy_right, rs_dz); +ALLY_DEADZONES(axis_z_left, lt_dz); +ALLY_DEADZONES(axis_z_right, rt_dz); + +/* ANTI-DEADZONES *********************************************************************************/ +static ssize_t _gamepad_apply_js_ADZ(struct hid_device *hdev, + struct ally_gamepad_cfg *ally_cfg) +{ + u8 *hidbuf; + int ret; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + hidbuf[2] = xpad_cmd_set_adz; + hidbuf[3] = xpad_cmd_len_adz; + hidbuf[4] = ally_cfg->ls_adz; + hidbuf[5] = ally_cfg->rs_adz; + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + goto report_fail; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + goto report_fail; + +report_fail: + kfree(hidbuf); + return ret; +} + +static void _gamepad_set_anti_deadzones_default(struct ally_gamepad_cfg *ally_cfg) +{ + ally_cfg->ls_adz = 0x00; + ally_cfg->rs_adz = 0x00; +} + +static ssize_t _gamepad_js_ADZ_store(struct device *dev, const char *buf, u8 *adz) +{ + int ret, val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < 0 || val > 32) + return -EINVAL; + + *adz = val; + + return ret; +} + +static ssize_t axis_xy_left_anti_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + + return sysfs_emit(buf, "%d\n", ally_cfg->ls_adz); +} + +static ssize_t axis_xy_left_anti_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + int ret; + + ret = _gamepad_js_ADZ_store(dev, buf, &ally_cfg->ls_adz); + if (ret) + return ret; + + return count; +} +ALLY_DEVICE_ATTR_RW(axis_xy_left_anti_deadzone, anti_deadzone); + +static ssize_t axis_xy_right_anti_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + + return sysfs_emit(buf, "%d\n", ally_cfg->rs_adz); +} + +static ssize_t axis_xy_right_anti_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + int ret; + + ret = _gamepad_js_ADZ_store(dev, buf, &ally_cfg->rs_adz); + if (ret) + return ret; + + return count; +} +ALLY_DEVICE_ATTR_RW(axis_xy_right_anti_deadzone, anti_deadzone); + +/* JS RESPONSE CURVES *****************************************************************************/ +static void _gamepad_set_js_response_curves_default(struct ally_gamepad_cfg *ally_cfg) +{ + struct response_curve *js1_rc = &ally_cfg->ls_rc; + struct response_curve *js2_rc = &ally_cfg->rs_rc; + js1_rc->move_pct_1 = js2_rc->move_pct_1 = 0x16; // 25% + js1_rc->move_pct_2 = js2_rc->move_pct_2 = 0x32; // 50% + js1_rc->move_pct_3 = js2_rc->move_pct_3 = 0x48; // 75% + js1_rc->move_pct_4 = js2_rc->move_pct_4 = 0x64; // 100% + js1_rc->response_pct_1 = js2_rc->response_pct_1 = 0x16; + js1_rc->response_pct_2 = js2_rc->response_pct_2 = 0x32; + js1_rc->response_pct_3 = js2_rc->response_pct_3 = 0x48; + js1_rc->response_pct_4 = js2_rc->response_pct_4 = 0x64; +} + +static ssize_t _gamepad_apply_response_curves(struct hid_device *hdev, + struct ally_gamepad_cfg *ally_cfg) +{ + u8 *hidbuf; + int ret; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + memcpy(&hidbuf[2], &ally_cfg->ls_rc, sizeof(ally_cfg->ls_rc)); + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + goto report_fail; + + hidbuf[4] = 0x02; + memcpy(&hidbuf[5], &ally_cfg->rs_rc, sizeof(ally_cfg->rs_rc)); + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + goto report_fail; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + goto report_fail; + +report_fail: + kfree(hidbuf); + return ret; +} + +ALLY_JS_RC_POINT(axis_xy_left, move, 1); +ALLY_JS_RC_POINT(axis_xy_left, move, 2); +ALLY_JS_RC_POINT(axis_xy_left, move, 3); +ALLY_JS_RC_POINT(axis_xy_left, move, 4); +ALLY_JS_RC_POINT(axis_xy_left, response, 1); +ALLY_JS_RC_POINT(axis_xy_left, response, 2); +ALLY_JS_RC_POINT(axis_xy_left, response, 3); +ALLY_JS_RC_POINT(axis_xy_left, response, 4); + +ALLY_JS_RC_POINT(axis_xy_right, move, 1); +ALLY_JS_RC_POINT(axis_xy_right, move, 2); +ALLY_JS_RC_POINT(axis_xy_right, move, 3); +ALLY_JS_RC_POINT(axis_xy_right, move, 4); +ALLY_JS_RC_POINT(axis_xy_right, response, 1); +ALLY_JS_RC_POINT(axis_xy_right, response, 2); +ALLY_JS_RC_POINT(axis_xy_right, response, 3); +ALLY_JS_RC_POINT(axis_xy_right, response, 4); + +/* CALIBRATIONS ***********************************************************************************/ +static int gamepad_get_calibration(struct hid_device *hdev) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + u8 *hidbuf; + int ret, i; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + for (i = 0; i < 2; i++) { + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = 0xD0; + hidbuf[2] = 0x03; + hidbuf[3] = i + 1; // 0x01 JS, 0x02 TR + hidbuf[4] = 0x20; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) { + hid_warn(hdev, "ROG Ally check failed set report: %d\n", ret); + goto cleanup; + } + + memset(hidbuf, 0, FEATURE_ROG_ALLY_REPORT_SIZE); + ret = asus_dev_get_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0 || hidbuf[5] != 1) { + hid_warn(hdev, "ROG Ally check failed get report: %d\n", ret); + goto cleanup; + } + + if (i == 0) { + /* Joystick calibration */ + reverse_bytes_in_pairs(&hidbuf[6], sizeof(struct js_axis_calibrations)); + ally_cfg->js_cal = *(struct js_axis_calibrations *)&hidbuf[6]; + print_hex_dump(KERN_INFO, "HID Buffer JS: ", DUMP_PREFIX_OFFSET, 16, 1, hidbuf, 32, true); + struct js_axis_calibrations *cal = &drvdata.gamepad_cfg->js_cal; + pr_err("LS_CAL: X: %d, Min: %d, Max: %d", cal->left_x_stable, cal->left_x_min, cal->left_x_max); + pr_err("LS_CAL: Y: %d, Min: %d, Max: %d", cal->left_y_stable, cal->left_y_min, cal->left_y_max); + pr_err("RS_CAL: X: %d, Min: %d, Max: %d", cal->right_x_stable, cal->right_x_min, cal->right_x_max); + pr_err("RS_CAL: Y: %d, Min: %d, Max: %d", cal->right_y_stable, cal->right_y_min, cal->right_y_max); + } else { + /* Trigger calibration */ + reverse_bytes_in_pairs(&hidbuf[6], sizeof(struct tr_axis_calibrations)); + ally_cfg->tr_cal = *(struct tr_axis_calibrations *)&hidbuf[6]; + print_hex_dump(KERN_INFO, "HID Buffer TR: ", DUMP_PREFIX_OFFSET, 16, 1, hidbuf, 32, true); + } + } + +cleanup: + kfree(hidbuf); + return ret; +} + +static struct attribute *axis_xy_left_attrs[] = { + &dev_attr_axis_xy_left_anti_deadzone.attr, + &dev_attr_axis_xy_left_deadzone.attr, + &dev_attr_axis_xyz_deadzone_index.attr, + &dev_attr_axis_xy_left_move_1.attr, + &dev_attr_axis_xy_left_move_2.attr, + &dev_attr_axis_xy_left_move_3.attr, + &dev_attr_axis_xy_left_move_4.attr, + &dev_attr_axis_xy_left_response_1.attr, + &dev_attr_axis_xy_left_response_2.attr, + &dev_attr_axis_xy_left_response_3.attr, + &dev_attr_axis_xy_left_response_4.attr, + NULL +}; +static const struct attribute_group axis_xy_left_attr_group = { + .name = "axis_xy_left", + .attrs = axis_xy_left_attrs, +}; + +static struct attribute *axis_xy_right_attrs[] = { + &dev_attr_axis_xy_right_anti_deadzone.attr, + &dev_attr_axis_xy_right_deadzone.attr, + &dev_attr_axis_xyz_deadzone_index.attr, + &dev_attr_axis_xy_right_move_1.attr, + &dev_attr_axis_xy_right_move_2.attr, + &dev_attr_axis_xy_right_move_3.attr, + &dev_attr_axis_xy_right_move_4.attr, + &dev_attr_axis_xy_right_response_1.attr, + &dev_attr_axis_xy_right_response_2.attr, + &dev_attr_axis_xy_right_response_3.attr, + &dev_attr_axis_xy_right_response_4.attr, + NULL +}; +static const struct attribute_group axis_xy_right_attr_group = { + .name = "axis_xy_right", + .attrs = axis_xy_right_attrs, +}; + +static struct attribute *axis_z_left_attrs[] = { + &dev_attr_axis_z_left_deadzone.attr, + &dev_attr_axis_xyz_deadzone_index.attr, + NULL, +}; +static const struct attribute_group axis_z_left_attr_group = { + .name = "axis_z_left", + .attrs = axis_z_left_attrs, +}; + +static struct attribute *axis_z_right_attrs[] = { + &dev_attr_axis_z_right_deadzone.attr, + &dev_attr_axis_xyz_deadzone_index.attr, + NULL, +}; +static const struct attribute_group axis_z_right_attr_group = { + .name = "axis_z_right", + .attrs = axis_z_right_attrs, +}; + +/* A HID packet conatins mappings for two buttons: btn1, btn1_macro, btn2, btn2_macro */ +static void _btn_pair_to_hid_pkt(struct ally_gamepad_cfg *ally_cfg, + enum btn_pair_index pair, + struct btn_data *btn1, struct btn_data *btn2, + u8 *out, int out_len) +{ + int start = 5; + + out[0] = FEATURE_ROG_ALLY_REPORT_ID; + out[1] = FEATURE_ROG_ALLY_CODE_PAGE; + out[2] = xpad_cmd_set_mapping; + out[3] = pair; + out[4] = xpad_cmd_len_mapping; + + btn_code_to_byte_array(btn1->button, &out[start]); + start += BTN_DATA_LEN; + btn_code_to_byte_array(btn1->macro, &out[start]); + start += BTN_DATA_LEN; + btn_code_to_byte_array(btn2->button, &out[start]); + start += BTN_DATA_LEN; + btn_code_to_byte_array(btn2->macro, &out[start]); + //print_hex_dump(KERN_DEBUG, "byte_array: ", DUMP_PREFIX_OFFSET, 64, 1, out, 64, false); +} + +/* Apply the mapping pair to the device */ +static int _gamepad_apply_btn_pair(struct hid_device *hdev, struct ally_gamepad_cfg *ally_cfg, + enum btn_pair_index btn_pair) +{ + u8 mode = ally_cfg->mode - 1; + struct btn_data *btn1, *btn2; + u8 *hidbuf; + int ret; + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + return ret; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + switch (btn_pair) { + case btn_pair_dpad_u_d: + btn1 = &ally_cfg->key_mapping[mode].dpad_up; + btn2 = &ally_cfg->key_mapping[mode].dpad_down; + break; + case btn_pair_dpad_l_r: + btn1 = &ally_cfg->key_mapping[mode].dpad_left; + btn2 = &ally_cfg->key_mapping[mode].dpad_right; + break; + case btn_pair_ls_rs: + btn1 = &ally_cfg->key_mapping[mode].btn_ls; + btn2 = &ally_cfg->key_mapping[mode].btn_rs; + break; + case btn_pair_lb_rb: + btn1 = &ally_cfg->key_mapping[mode].btn_lb; + btn2 = &ally_cfg->key_mapping[mode].btn_rb; + break; + case btn_pair_lt_rt: + btn1 = &ally_cfg->key_mapping[mode].btn_lt; + btn2 = &ally_cfg->key_mapping[mode].btn_rt; + break; + case btn_pair_a_b: + btn1 = &ally_cfg->key_mapping[mode].btn_a; + btn2 = &ally_cfg->key_mapping[mode].btn_b; + break; + case btn_pair_x_y: + btn1 = &ally_cfg->key_mapping[mode].btn_x; + btn2 = &ally_cfg->key_mapping[mode].btn_y; + break; + case btn_pair_view_menu: + btn1 = &ally_cfg->key_mapping[mode].btn_view; + btn2 = &ally_cfg->key_mapping[mode].btn_menu; + break; + case btn_pair_m1_m2: + btn1 = &ally_cfg->key_mapping[mode].btn_m1; + btn2 = &ally_cfg->key_mapping[mode].btn_m2; + break; + default: + break; + } + + _btn_pair_to_hid_pkt(ally_cfg, btn_pair, btn1, btn2, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + + kfree(hidbuf); + + return ret; +} + +static int _gamepad_apply_turbo(struct hid_device *hdev, struct ally_gamepad_cfg *ally_cfg) +{ + struct btn_mapping *map = &ally_cfg->key_mapping[ally_cfg->mode - 1]; + u8 *hidbuf; + int ret; + + /* set turbo */ + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + hidbuf[2] = xpad_cmd_set_turbo; + hidbuf[3] = xpad_cmd_len_turbo; + + hidbuf[4] = map->dpad_up.turbo; + hidbuf[6] = map->dpad_down.turbo; + hidbuf[8] = map->dpad_left.turbo; + hidbuf[10] = map->dpad_right.turbo; + + hidbuf[12] = map->btn_ls.turbo; + hidbuf[14] = map->btn_rs.turbo; + hidbuf[16] = map->btn_lb.turbo; + hidbuf[18] = map->btn_rb.turbo; + + hidbuf[20] = map->btn_a.turbo; + hidbuf[22] = map->btn_b.turbo; + hidbuf[24] = map->btn_x.turbo; + hidbuf[26] = map->btn_y.turbo; + + hidbuf[28] = map->btn_lt.turbo; + hidbuf[30] = map->btn_rt.turbo; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + + kfree(hidbuf); + + return ret; +} + +static ssize_t _gamepad_apply_all(struct hid_device *hdev, struct ally_gamepad_cfg *ally_cfg) +{ + int ret; + + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_dpad_u_d); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_dpad_l_r); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_ls_rs); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_lb_rb); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_a_b); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_x_y); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_view_menu); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_m1_m2); + if (ret < 0) + return ret; + ret = _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_lt_rt); + if (ret < 0) + return ret; + ret = _gamepad_apply_turbo(hdev, ally_cfg); + if (ret < 0) + return ret; + ret = _gamepad_apply_deadzones(hdev, ally_cfg); + if (ret < 0) + return ret; + ret = _gamepad_apply_js_ADZ(hdev, ally_cfg); + if (ret < 0) + return ret; + ret =_gamepad_apply_response_curves(hdev, ally_cfg); + if (ret < 0) + return ret; + + return 0; +} + +static ssize_t gamepad_apply_all_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + struct hid_device *hdev = to_hid_device(dev); + int ret; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + ret = _gamepad_apply_all(hdev, ally_cfg); + if (ret < 0) + return ret; + + return count; +} +ALLY_DEVICE_ATTR_WO(gamepad_apply_all, apply_all); + +/* button map attributes, regular and macro*/ +ALLY_BTN_MAPPING(m1, btn_m1); +ALLY_BTN_MAPPING(m2, btn_m2); +ALLY_BTN_MAPPING(view, btn_view); +ALLY_BTN_MAPPING(menu, btn_menu); +ALLY_TURBO_BTN_MAPPING(a, btn_a); +ALLY_TURBO_BTN_MAPPING(b, btn_b); +ALLY_TURBO_BTN_MAPPING(x, btn_x); +ALLY_TURBO_BTN_MAPPING(y, btn_y); +ALLY_TURBO_BTN_MAPPING(lb, btn_lb); +ALLY_TURBO_BTN_MAPPING(rb, btn_rb); +ALLY_TURBO_BTN_MAPPING(ls, btn_ls); +ALLY_TURBO_BTN_MAPPING(rs, btn_rs); +ALLY_TURBO_BTN_MAPPING(lt, btn_lt); +ALLY_TURBO_BTN_MAPPING(rt, btn_rt); +ALLY_TURBO_BTN_MAPPING(dpad_u, dpad_up); +ALLY_TURBO_BTN_MAPPING(dpad_d, dpad_down); +ALLY_TURBO_BTN_MAPPING(dpad_l, dpad_left); +ALLY_TURBO_BTN_MAPPING(dpad_r, dpad_right); + +static void _gamepad_set_xpad_default(struct ally_gamepad_cfg *ally_cfg) +{ + struct btn_mapping *map = &ally_cfg->key_mapping[ally_cfg->mode - 1]; + map->btn_m1.button = BTN_KB_M1; + map->btn_m2.button = BTN_KB_M2; + map->btn_a.button = BTN_PAD_A; + map->btn_b.button = BTN_PAD_B; + map->btn_x.button = BTN_PAD_X; + map->btn_y.button = BTN_PAD_Y; + map->btn_lb.button = BTN_PAD_LB; + map->btn_rb.button = BTN_PAD_RB; + map->btn_lt.button = BTN_PAD_LT; + map->btn_rt.button = BTN_PAD_RT; + map->btn_ls.button = BTN_PAD_LS; + map->btn_rs.button = BTN_PAD_RS; + map->dpad_up.button = BTN_PAD_DPAD_UP; + map->dpad_down.button = BTN_PAD_DPAD_DOWN; + map->dpad_left.button = BTN_PAD_DPAD_LEFT; + map->dpad_right.button = BTN_PAD_DPAD_RIGHT; + map->btn_view.button = BTN_PAD_VIEW; + map->btn_menu.button = BTN_PAD_MENU; +} + +static ssize_t btn_mapping_reset_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + switch (ally_cfg->mode) { + case xpad_mode_game: + _gamepad_set_xpad_default(ally_cfg); + break; + default: + _gamepad_set_xpad_default(ally_cfg); + break; + } + + return count; +} +ALLY_DEVICE_ATTR_WO(btn_mapping_reset, reset_btn_mapping); + +/* GAMEPAD MODE */ +static ssize_t _gamepad_set_mode(struct hid_device *hdev, struct ally_gamepad_cfg *ally_cfg, + int val) +{ + u8 *hidbuf; + int ret; + + hidbuf = kzalloc(FEATURE_ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + hidbuf[0] = FEATURE_ROG_ALLY_REPORT_ID; + hidbuf[1] = FEATURE_ROG_ALLY_CODE_PAGE; + hidbuf[2] = xpad_cmd_set_mode; + hidbuf[3] = xpad_cmd_len_mode; + hidbuf[4] = val; + + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + goto report_fail; + + ret = asus_dev_set_report(hdev, hidbuf, FEATURE_ROG_ALLY_REPORT_SIZE); + if (ret < 0) + goto report_fail; + + ret = _gamepad_apply_all(hdev, ally_cfg); + if (ret < 0) + goto report_fail; + +report_fail: + kfree(hidbuf); + return ret; +} + +static ssize_t gamepad_mode_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + return sysfs_emit(buf, "%d\n", ally_cfg->mode); +} + +static ssize_t gamepad_mode_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + int ret, val; + + if (!drvdata.gamepad_cfg) + return -ENODEV; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < xpad_mode_game || val > xpad_mode_mouse) + return -EINVAL; + + ally_cfg->mode = val; + + ret = _gamepad_set_mode(hdev, ally_cfg, val); + if (ret < 0) + return ret; + + return count; +} + +DEVICE_ATTR_RW(gamepad_mode); + +static ssize_t mcu_version_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", drvdata.mcu_version); +} + +DEVICE_ATTR_RO(mcu_version); + +/* ROOT LEVEL ATTRS *******************************************************************************/ +static struct attribute *gamepad_device_attrs[] = { + &dev_attr_btn_mapping_reset.attr, + &dev_attr_gamepad_mode.attr, + &dev_attr_gamepad_apply_all.attr, + &dev_attr_gamepad_vibration_intensity.attr, + &dev_attr_gamepad_vibration_intensity_index.attr, + &dev_attr_mcu_version.attr, + NULL +}; + +static const struct attribute_group ally_controller_attr_group = { + .attrs = gamepad_device_attrs, +}; + +static const struct attribute_group *gamepad_device_attr_groups[] = { + &ally_controller_attr_group, + &axis_xy_left_attr_group, + &axis_xy_right_attr_group, + &axis_z_left_attr_group, + &axis_z_right_attr_group, + &btn_mapping_m1_attr_group, + &btn_mapping_m2_attr_group, + &btn_mapping_a_attr_group, + &btn_mapping_b_attr_group, + &btn_mapping_x_attr_group, + &btn_mapping_y_attr_group, + &btn_mapping_lb_attr_group, + &btn_mapping_rb_attr_group, + &btn_mapping_ls_attr_group, + &btn_mapping_rs_attr_group, + &btn_mapping_lt_attr_group, + &btn_mapping_rt_attr_group, + &btn_mapping_dpad_u_attr_group, + &btn_mapping_dpad_d_attr_group, + &btn_mapping_dpad_l_attr_group, + &btn_mapping_dpad_r_attr_group, + &btn_mapping_view_attr_group, + &btn_mapping_menu_attr_group, + NULL, +}; + +static struct ally_gamepad_cfg *ally_gamepad_cfg_create(struct hid_device *hdev) +{ + struct ally_gamepad_cfg *ally_cfg; + struct input_dev *input_dev; + int err; + + ally_cfg = devm_kzalloc(&hdev->dev, sizeof(*ally_cfg), GFP_KERNEL); + if (!ally_cfg) + return ERR_PTR(-ENOMEM); + ally_cfg->hdev = hdev; + // Allocate memory for each mode's `btn_mapping` + ally_cfg->mode = xpad_mode_game; + + input_dev = devm_input_allocate_device(&hdev->dev); + if (!input_dev) { + err = -ENOMEM; + goto free_ally_cfg; + } + + input_dev->id.bustype = hdev->bus; + input_dev->id.vendor = hdev->vendor; + input_dev->id.product = hdev->product; + input_dev->id.version = hdev->version; + input_dev->uniq = hdev->uniq; + input_dev->name = "ASUS ROG Ally Config"; + input_set_capability(input_dev, EV_KEY, KEY_PROG1); + input_set_capability(input_dev, EV_KEY, KEY_F16); + input_set_capability(input_dev, EV_KEY, KEY_F17); + input_set_capability(input_dev, EV_KEY, KEY_F18); + input_set_drvdata(input_dev, hdev); + + err = input_register_device(input_dev); + if (err) + goto free_input_dev; + ally_cfg->input = input_dev; + + /* ignore all errors for this as they are related to USB HID I/O */ + _gamepad_set_xpad_default(ally_cfg); + ally_cfg->key_mapping[ally_cfg->mode - 1].btn_m1.button = BTN_KB_M1; + ally_cfg->key_mapping[ally_cfg->mode - 1].btn_m2.button = BTN_KB_M2; + _gamepad_apply_btn_pair(hdev, ally_cfg, btn_pair_m1_m2); + gamepad_get_calibration(hdev); + + ally_cfg->vibration_intensity[0] = 0x64; + ally_cfg->vibration_intensity[1] = 0x64; + _gamepad_set_deadzones_default(ally_cfg); + _gamepad_set_anti_deadzones_default(ally_cfg); + _gamepad_set_js_response_curves_default(ally_cfg); + + drvdata.gamepad_cfg = ally_cfg; // Must asign before attr group setup + if (sysfs_create_groups(&hdev->dev.kobj, gamepad_device_attr_groups)) { + err = -ENODEV; + goto unregister_input_dev; + } + + return ally_cfg; + +unregister_input_dev: + input_unregister_device(input_dev); + ally_cfg->input = NULL; // Prevent double free when kfree(ally_cfg) happens + +free_input_dev: + devm_kfree(&hdev->dev, input_dev); + +free_ally_cfg: + devm_kfree(&hdev->dev, ally_cfg); + return ERR_PTR(err); +} + +static void ally_cfg_remove(struct hid_device *hdev) +{ + // __gamepad_set_mode(hdev, drvdata.gamepad_cfg, xpad_mode_mouse); + sysfs_remove_groups(&hdev->dev.kobj, gamepad_device_attr_groups); +} + +/**************************************************************************************************/ +/* ROG Ally gamepad i/o and force-feedback */ +/**************************************************************************************************/ +static int ally_x_raw_event(struct ally_x_device *ally_x, struct hid_report *report, u8 *data, + int size) +{ + struct ally_x_input_report *in_report; + unsigned long flags; + u8 byte; + + if (data[0] == 0x0B) { + in_report = (struct ally_x_input_report *)&data[1]; + + input_report_abs(ally_x->input, ABS_X, in_report->x); + input_report_abs(ally_x->input, ABS_Y, in_report->y); + input_report_abs(ally_x->input, ABS_RX, in_report->rx); + input_report_abs(ally_x->input, ABS_RY, in_report->ry); + input_report_abs(ally_x->input, ABS_Z, in_report->z); + input_report_abs(ally_x->input, ABS_RZ, in_report->rz); + + byte = in_report->buttons[0]; + input_report_key(ally_x->input, BTN_A, byte & BIT(0)); + input_report_key(ally_x->input, BTN_B, byte & BIT(1)); + input_report_key(ally_x->input, BTN_X, byte & BIT(2)); + input_report_key(ally_x->input, BTN_Y, byte & BIT(3)); + input_report_key(ally_x->input, BTN_TL, byte & BIT(4)); + input_report_key(ally_x->input, BTN_TR, byte & BIT(5)); + input_report_key(ally_x->input, BTN_SELECT, byte & BIT(6)); + input_report_key(ally_x->input, BTN_START, byte & BIT(7)); + + byte = in_report->buttons[1]; + input_report_key(ally_x->input, BTN_THUMBL, byte & BIT(0)); + input_report_key(ally_x->input, BTN_THUMBR, byte & BIT(1)); + input_report_key(ally_x->input, BTN_MODE, byte & BIT(2)); + + byte = in_report->buttons[2]; + input_report_abs(ally_x->input, ABS_HAT0X, hat_values[byte][0]); + input_report_abs(ally_x->input, ABS_HAT0Y, hat_values[byte][1]); + } + /* + * The MCU used on Ally provides many devices: gamepad, keyboord, mouse, other. + * The AC and QAM buttons route through another interface making it difficult to + * use the events unless we grab those and use them here. Only works for Ally X. + */ + else if (data[0] == 0x5A) { + if (ally_x->qam_btns_steam_mode) { + spin_lock_irqsave(&ally_x->lock, flags); + if (data[1] == 0x38 && !ally_x->update_qam_btn) { + ally_x->update_qam_btn = true; + if (ally_x->output_worker_initialized) + schedule_work(&ally_x->output_worker); + } + spin_unlock_irqrestore(&ally_x->lock, flags); + /* Left/XBox button. Long press does ctrl+alt+del which we can't catch */ + input_report_key(ally_x->input, BTN_MODE, data[1] == 0xA6); + } else { + input_report_key(ally_x->input, KEY_F16, data[1] == 0xA6); + input_report_key(ally_x->input, KEY_PROG1, data[1] == 0x38); + } + /* QAM long press */ + input_report_key(ally_x->input, KEY_F17, data[1] == 0xA7); + /* QAM long press released */ + input_report_key(ally_x->input, KEY_F18, data[1] == 0xA8); + } + + input_sync(ally_x->input); + + return 0; +} + +static struct input_dev *ally_x_alloc_input_dev(struct hid_device *hdev, + const char *name_suffix) +{ + struct input_dev *input_dev; + + input_dev = devm_input_allocate_device(&hdev->dev); + if (!input_dev) + return ERR_PTR(-ENOMEM); + + input_dev->id.bustype = hdev->bus; + input_dev->id.vendor = hdev->vendor; + input_dev->id.product = hdev->product; + input_dev->id.version = hdev->version; + input_dev->uniq = hdev->uniq; + input_dev->name = "ASUS ROG Ally X Gamepad"; + + input_set_drvdata(input_dev, hdev); + + return input_dev; +} + +static int ally_x_play_effect(struct input_dev *idev, void *data, struct ff_effect *effect) +{ + struct ally_x_device *ally_x = drvdata.ally_x; + unsigned long flags; + + if (effect->type != FF_RUMBLE) + return 0; + + spin_lock_irqsave(&ally_x->lock, flags); + ally_x->ff_packet->ff.magnitude_strong = effect->u.rumble.strong_magnitude / 512; + ally_x->ff_packet->ff.magnitude_weak = effect->u.rumble.weak_magnitude / 512; + ally_x->update_ff = true; + spin_unlock_irqrestore(&ally_x->lock, flags); + + if (ally_x->output_worker_initialized) + schedule_work(&ally_x->output_worker); + + return 0; +} + +static void ally_x_work(struct work_struct *work) +{ + struct ally_x_device *ally_x = container_of(work, struct ally_x_device, output_worker); + struct ff_report *ff_report = NULL; + bool update_qam = false; + bool update_ff = false; + unsigned long flags; + + spin_lock_irqsave(&ally_x->lock, flags); + update_ff = ally_x->update_ff; + if (ally_x->update_ff) { + ff_report = kmemdup(ally_x->ff_packet, sizeof(*ally_x->ff_packet), GFP_KERNEL); + ally_x->update_ff = false; + } + update_qam = ally_x->update_qam_btn; + spin_unlock_irqrestore(&ally_x->lock, flags); + + if (update_ff && ff_report) { + ff_report->ff.magnitude_left = ff_report->ff.magnitude_strong; + ff_report->ff.magnitude_right = ff_report->ff.magnitude_weak; + asus_dev_set_report(ally_x->hdev, (u8 *)ff_report, sizeof(*ff_report)); + } + kfree(ff_report); + + if (update_qam) { + /* + * The sleeps here are required to allow steam to register the button combo. + */ + usleep_range(1000, 2000); + input_report_key(ally_x->input, BTN_MODE, 1); + input_sync(ally_x->input); + + msleep(80); + input_report_key(ally_x->input, BTN_A, 1); + input_sync(ally_x->input); + + msleep(80); + input_report_key(ally_x->input, BTN_A, 0); + input_sync(ally_x->input); + + msleep(80); + input_report_key(ally_x->input, BTN_MODE, 0); + input_sync(ally_x->input); + + spin_lock_irqsave(&ally_x->lock, flags); + ally_x->update_qam_btn = false; + spin_unlock_irqrestore(&ally_x->lock, flags); + } +} + +static struct input_dev *ally_x_setup_input(struct hid_device *hdev) +{ + int ret, abs_min = 0, js_abs_max = 65535, tr_abs_max = 1023; + struct input_dev *input; + + input = ally_x_alloc_input_dev(hdev, NULL); + if (IS_ERR(input)) + return ERR_CAST(input); + + input_set_abs_params(input, ABS_X, abs_min, js_abs_max, 0, 0); + input_set_abs_params(input, ABS_Y, abs_min, js_abs_max, 0, 0); + input_set_abs_params(input, ABS_RX, abs_min, js_abs_max, 0, 0); + input_set_abs_params(input, ABS_RY, abs_min, js_abs_max, 0, 0); + input_set_abs_params(input, ABS_Z, abs_min, tr_abs_max, 0, 0); + input_set_abs_params(input, ABS_RZ, abs_min, tr_abs_max, 0, 0); + input_set_abs_params(input, ABS_HAT0X, -1, 1, 0, 0); + input_set_abs_params(input, ABS_HAT0Y, -1, 1, 0, 0); + input_set_capability(input, EV_KEY, BTN_A); + input_set_capability(input, EV_KEY, BTN_B); + input_set_capability(input, EV_KEY, BTN_X); + input_set_capability(input, EV_KEY, BTN_Y); + input_set_capability(input, EV_KEY, BTN_TL); + input_set_capability(input, EV_KEY, BTN_TR); + input_set_capability(input, EV_KEY, BTN_SELECT); + input_set_capability(input, EV_KEY, BTN_START); + input_set_capability(input, EV_KEY, BTN_MODE); + input_set_capability(input, EV_KEY, BTN_THUMBL); + input_set_capability(input, EV_KEY, BTN_THUMBR); + + input_set_capability(input, EV_KEY, KEY_PROG1); + input_set_capability(input, EV_KEY, KEY_F16); + input_set_capability(input, EV_KEY, KEY_F17); + input_set_capability(input, EV_KEY, KEY_F18); + + input_set_capability(input, EV_FF, FF_RUMBLE); + input_ff_create_memless(input, NULL, ally_x_play_effect); + + ret = input_register_device(input); + if (ret) + return ERR_PTR(ret); + + return input; +} + +static ssize_t ally_x_qam_mode_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ally_x_device *ally_x = drvdata.ally_x; + + return sysfs_emit(buf, "%d\n", ally_x->qam_btns_steam_mode); +} + +static ssize_t ally_x_qam_mode_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ally_x_device *ally_x = drvdata.ally_x; + bool val; + int ret; + + ret = kstrtobool(buf, &val); + if (ret < 0) + return ret; + + ally_x->qam_btns_steam_mode = val; + + return count; +} +ALLY_DEVICE_ATTR_RW(ally_x_qam_mode, qam_mode); + +static struct ally_x_device *ally_x_create(struct hid_device *hdev) +{ + uint8_t max_output_report_size; + struct ally_x_device *ally_x; + struct ff_report *report; + int ret; + + ally_x = devm_kzalloc(&hdev->dev, sizeof(*ally_x), GFP_KERNEL); + if (!ally_x) + return ERR_PTR(-ENOMEM); + + ally_x->hdev = hdev; + INIT_WORK(&ally_x->output_worker, ally_x_work); + spin_lock_init(&ally_x->lock); + ally_x->output_worker_initialized = true; + ally_x->qam_btns_steam_mode = + true; /* Always default to steam mode, it can be changed by userspace attr */ + + max_output_report_size = sizeof(struct ally_x_input_report); + report = devm_kzalloc(&hdev->dev, sizeof(*report), GFP_KERNEL); + if (!report) { + ret = -ENOMEM; + goto free_ally_x; + } + + /* None of these bytes will change for the FF command for now */ + report->report_id = 0x0D; + report->ff.enable = 0x0F; /* Enable all by default */ + report->ff.pulse_sustain_10ms = 0xFF; /* Duration */ + report->ff.pulse_release_10ms = 0x00; /* Start Delay */ + report->ff.loop_count = 0xEB; /* Loop Count */ + ally_x->ff_packet = report; + + ally_x->input = ally_x_setup_input(hdev); + if (IS_ERR(ally_x->input)) { + ret = PTR_ERR(ally_x->input); + goto free_ff_packet; + } + + if (sysfs_create_file(&hdev->dev.kobj, &dev_attr_ally_x_qam_mode.attr)) { + ret = -ENODEV; + goto unregister_input; + } + + ally_x->update_ff = true; + if (ally_x->output_worker_initialized) + schedule_work(&ally_x->output_worker); + + hid_info(hdev, "Registered Ally X controller using %s\n", + dev_name(&ally_x->input->dev)); + return ally_x; + +unregister_input: + input_unregister_device(ally_x->input); +free_ff_packet: + kfree(ally_x->ff_packet); +free_ally_x: + kfree(ally_x); + return ERR_PTR(ret); +} + +static void ally_x_remove(struct hid_device *hdev) +{ + struct ally_x_device *ally_x = drvdata.ally_x; + unsigned long flags; + + spin_lock_irqsave(&ally_x->lock, flags); + ally_x->output_worker_initialized = false; + spin_unlock_irqrestore(&ally_x->lock, flags); + cancel_work_sync(&ally_x->output_worker); + sysfs_remove_file(&hdev->dev.kobj, &dev_attr_ally_x_qam_mode.attr); +} + +/**************************************************************************************************/ +/* ROG Ally LED control */ +/**************************************************************************************************/ +static void ally_rgb_schedule_work(struct ally_rgb_dev *led) +{ + unsigned long flags; + + spin_lock_irqsave(&led->lock, flags); + if (!led->removed) + schedule_work(&led->work); + spin_unlock_irqrestore(&led->lock, flags); +} + +/* + * The RGB still has the basic 0-3 level brightness. Since the multicolour + * brightness is being used in place, set this to max + */ +static int ally_rgb_set_bright_base_max(struct hid_device *hdev) +{ + u8 buf[] = { FEATURE_KBD_LED_REPORT_ID1, 0xba, 0xc5, 0xc4, 0x02 }; + + return asus_dev_set_report(hdev, buf, sizeof(buf)); +} + +static void ally_rgb_do_work(struct work_struct *work) +{ + struct ally_rgb_dev *led = container_of(work, struct ally_rgb_dev, work); + int ret; + unsigned long flags; + + u8 buf[16] = { [0] = FEATURE_ROG_ALLY_REPORT_ID, + [1] = FEATURE_ROG_ALLY_CODE_PAGE, + [2] = xpad_cmd_set_leds, + [3] = xpad_cmd_len_leds }; + + spin_lock_irqsave(&led->lock, flags); + if (!led->update_rgb) { + spin_unlock_irqrestore(&led->lock, flags); + return; + } + + for (int i = 0; i < 4; i++) { + buf[5 + i * 3] = drvdata.led_rgb_dev->green[i]; + buf[6 + i * 3] = drvdata.led_rgb_dev->blue[i]; + buf[4 + i * 3] = drvdata.led_rgb_dev->red[i]; + } + led->update_rgb = false; + + spin_unlock_irqrestore(&led->lock, flags); + + ret = asus_dev_set_report(led->hdev, buf, sizeof(buf)); + if (ret < 0) + hid_err(led->hdev, "Ally failed to set gamepad backlight: %d\n", ret); +} + +static void ally_rgb_set(struct led_classdev *cdev, enum led_brightness brightness) +{ + struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(cdev); + struct ally_rgb_dev *led = container_of(mc_cdev, struct ally_rgb_dev, led_rgb_dev); + int intensity, bright; + unsigned long flags; + + led_mc_calc_color_components(mc_cdev, brightness); + spin_lock_irqsave(&led->lock, flags); + led->update_rgb = true; + bright = mc_cdev->led_cdev.brightness; + for (int i = 0; i < 4; i++) { + intensity = mc_cdev->subled_info[i].intensity; + drvdata.led_rgb_dev->red[i] = (((intensity >> 16) & 0xFF) * bright) / 255; + drvdata.led_rgb_dev->green[i] = (((intensity >> 8) & 0xFF) * bright) / 255; + drvdata.led_rgb_dev->blue[i] = ((intensity & 0xFF) * bright) / 255; + } + spin_unlock_irqrestore(&led->lock, flags); + drvdata.led_rgb_data.initialized = true; + + ally_rgb_schedule_work(led); +} + +static int ally_rgb_set_static_from_multi(struct hid_device *hdev) +{ + u8 buf[17] = {FEATURE_KBD_LED_REPORT_ID1, 0xb3}; + int ret; + + /* + * Set single zone single colour based on the first LED of EC software mode. + * buf[2] = zone, buf[3] = mode + */ + buf[4] = drvdata.led_rgb_data.red[0]; + buf[5] = drvdata.led_rgb_data.green[0]; + buf[6] = drvdata.led_rgb_data.blue[0]; + + ret = asus_dev_set_report(hdev, buf, sizeof(buf)); + if (ret < 0) + return ret; + + ret = asus_dev_set_report(hdev, EC_MODE_LED_APPLY, sizeof(EC_MODE_LED_APPLY)); + if (ret < 0) + return ret; + + return asus_dev_set_report(hdev, EC_MODE_LED_SET, sizeof(EC_MODE_LED_SET)); +} + +/* + * Store the RGB values for restoring on resume, and set the static mode to the first LED colour +*/ +static void ally_rgb_store_settings(void) +{ + int arr_size = sizeof(drvdata.led_rgb_data.red); + + struct ally_rgb_dev *led_rgb = drvdata.led_rgb_dev; + + drvdata.led_rgb_data.brightness = led_rgb->led_rgb_dev.led_cdev.brightness; + + memcpy(drvdata.led_rgb_data.red, led_rgb->red, arr_size); + memcpy(drvdata.led_rgb_data.green, led_rgb->green, arr_size); + memcpy(drvdata.led_rgb_data.blue, led_rgb->blue, arr_size); + + ally_rgb_set_static_from_multi(led_rgb->hdev); +} + +static void ally_rgb_restore_settings(struct ally_rgb_dev *led_rgb, struct led_classdev *led_cdev, + struct mc_subled *mc_led_info) +{ + int arr_size = sizeof(drvdata.led_rgb_data.red); + + memcpy(led_rgb->red, drvdata.led_rgb_data.red, arr_size); + memcpy(led_rgb->green, drvdata.led_rgb_data.green, arr_size); + memcpy(led_rgb->blue, drvdata.led_rgb_data.blue, arr_size); + for (int i = 0; i < 4; i++) { + mc_led_info[i].intensity = (drvdata.led_rgb_data.red[i] << 16) | + (drvdata.led_rgb_data.green[i] << 8) | + drvdata.led_rgb_data.blue[i]; + } + led_cdev->brightness = drvdata.led_rgb_data.brightness; +} + +/* Set LEDs. Call after any setup. */ +static void ally_rgb_resume(void) +{ + struct ally_rgb_dev *led_rgb = drvdata.led_rgb_dev; + struct led_classdev *led_cdev; + struct mc_subled *mc_led_info; + + if (!led_rgb) + return; + + led_cdev = &led_rgb->led_rgb_dev.led_cdev; + mc_led_info = led_rgb->led_rgb_dev.subled_info; + + if (drvdata.led_rgb_data.initialized) { + ally_rgb_restore_settings(led_rgb, led_cdev, mc_led_info); + led_rgb->update_rgb = true; + ally_rgb_schedule_work(led_rgb); + ally_rgb_set_bright_base_max(led_rgb->hdev); + } +} + +static int ally_rgb_register(struct hid_device *hdev, struct ally_rgb_dev *led_rgb) +{ + struct mc_subled *mc_led_info; + struct led_classdev *led_cdev; + + mc_led_info = + devm_kmalloc_array(&hdev->dev, 12, sizeof(*mc_led_info), GFP_KERNEL | __GFP_ZERO); + if (!mc_led_info) + return -ENOMEM; + + mc_led_info[0].color_index = LED_COLOR_ID_RGB; + mc_led_info[1].color_index = LED_COLOR_ID_RGB; + mc_led_info[2].color_index = LED_COLOR_ID_RGB; + mc_led_info[3].color_index = LED_COLOR_ID_RGB; + + led_rgb->led_rgb_dev.subled_info = mc_led_info; + led_rgb->led_rgb_dev.num_colors = 4; + + led_cdev = &led_rgb->led_rgb_dev.led_cdev; + led_cdev->brightness = 128; + led_cdev->name = "ally:rgb:joystick_rings"; + led_cdev->max_brightness = 255; + led_cdev->brightness_set = ally_rgb_set; + + if (drvdata.led_rgb_data.initialized) { + ally_rgb_restore_settings(led_rgb, led_cdev, mc_led_info); + } + + return devm_led_classdev_multicolor_register(&hdev->dev, &led_rgb->led_rgb_dev); +} + +static struct ally_rgb_dev *ally_rgb_create(struct hid_device *hdev) +{ + struct ally_rgb_dev *led_rgb; + int ret; + + led_rgb = devm_kzalloc(&hdev->dev, sizeof(struct ally_rgb_dev), GFP_KERNEL); + if (!led_rgb) + return ERR_PTR(-ENOMEM); + + ret = ally_rgb_register(hdev, led_rgb); + if (ret < 0) { + cancel_work_sync(&led_rgb->work); + devm_kfree(&hdev->dev, led_rgb); + return ERR_PTR(ret); + } + + led_rgb->hdev = hdev; + led_rgb->removed = false; + + INIT_WORK(&led_rgb->work, ally_rgb_do_work); + led_rgb->output_worker_initialized = true; + spin_lock_init(&led_rgb->lock); + + ally_rgb_set_bright_base_max(hdev); + + /* Not marked as initialized unless ally_rgb_set() is called */ + if (drvdata.led_rgb_data.initialized) { + msleep(1500); + led_rgb->update_rgb = true; + ally_rgb_schedule_work(led_rgb); + } + + return led_rgb; +} + +static void ally_rgb_remove(struct hid_device *hdev) +{ + struct ally_rgb_dev *led_rgb = drvdata.led_rgb_dev; + unsigned long flags; + int ep; + + ep = get_endpoint_address(hdev); + if (ep != ROG_ALLY_CFG_INTF_IN) + return; + + if (!drvdata.led_rgb_dev || led_rgb->removed) + return; + + spin_lock_irqsave(&led_rgb->lock, flags); + led_rgb->removed = true; + led_rgb->output_worker_initialized = false; + spin_unlock_irqrestore(&led_rgb->lock, flags); + cancel_work_sync(&led_rgb->work); + devm_led_classdev_multicolor_unregister(&hdev->dev, &led_rgb->led_rgb_dev); + + hid_info(hdev, "Removed Ally RGB interface"); +} + +/**************************************************************************************************/ +/* ROG Ally driver init */ +/**************************************************************************************************/ + +static int ally_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, + int size) +{ + struct ally_gamepad_cfg *cfg = drvdata.gamepad_cfg; + struct ally_x_device *ally_x = drvdata.ally_x; + + if (ally_x) { + if ((hdev->bus == BUS_USB && report->id == ALLY_X_INPUT_REPORT_USB && + size == ALLY_X_INPUT_REPORT_USB_SIZE) || + (data[0] == 0x5A)) { + ally_x_raw_event(ally_x, report, data, size); + } else { + return -1; + } + } + + if (cfg && !ally_x) { + input_report_key(cfg->input, KEY_PROG1, data[1] == 0x38); + input_report_key(cfg->input, KEY_F16, data[1] == 0xA6); + input_report_key(cfg->input, KEY_F17, data[1] == 0xA7); + input_report_key(cfg->input, KEY_F18, data[1] == 0xA8); + input_sync(cfg->input); + } + + return 0; +} + +static int ally_hid_init(struct hid_device *hdev) +{ + int ret; + + ret = asus_dev_set_report(hdev, EC_INIT_STRING, sizeof(EC_INIT_STRING)); + if (ret < 0) { + hid_err(hdev, "Ally failed to send init command: %d\n", ret); + return ret; + } + + ret = asus_dev_set_report(hdev, FORCE_FEEDBACK_OFF, sizeof(FORCE_FEEDBACK_OFF)); + if (ret < 0) + hid_err(hdev, "Ally failed to send init command: %d\n", ret); + + return ret; +} + +static int ally_hid_probe(struct hid_device *hdev, const struct hid_device_id *_id) +{ + struct usb_interface *intf = to_usb_interface(hdev->dev.parent); + struct usb_device *udev = interface_to_usbdev(intf); + u16 idProduct = le16_to_cpu(udev->descriptor.idProduct); + int ret, ep; + + ep = get_endpoint_address(hdev); + if (ep < 0) + return ep; + + if (ep != ROG_ALLY_CFG_INTF_IN && + ep != ROG_ALLY_X_INTF_IN) + return -ENODEV; + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "Parse failed\n"); + return ret; + } + + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + if (ret) { + hid_err(hdev, "Failed to start HID device\n"); + return ret; + } + + ret = hid_hw_open(hdev); + if (ret) { + hid_err(hdev, "Failed to open HID device\n"); + goto err_stop; + } + + /* Initialize MCU even before alloc */ + ret = ally_hid_init(hdev); + if (ret < 0) + return ret; + + drvdata.hdev = hdev; + hid_set_drvdata(hdev, &drvdata); + + /* This should almost always exist */ + if (ep == ROG_ALLY_CFG_INTF_IN) { + validate_mcu_fw_version(hdev, idProduct); + + drvdata.led_rgb_dev = ally_rgb_create(hdev); + if (IS_ERR(drvdata.led_rgb_dev)) + hid_err(hdev, "Failed to create Ally gamepad LEDs.\n"); + else + hid_info(hdev, "Created Ally RGB LED controls.\n"); + + drvdata.gamepad_cfg = ally_gamepad_cfg_create(hdev); + if (IS_ERR(drvdata.gamepad_cfg)) + hid_err(hdev, "Failed to create Ally gamepad attributes.\n"); + else + hid_info(hdev, "Created Ally gamepad attributes.\n"); + + if (IS_ERR(drvdata.led_rgb_dev) && IS_ERR(drvdata.gamepad_cfg)) + goto err_close; + } + + /* May or may not exist */ + if (ep == ROG_ALLY_X_INTF_IN) { + drvdata.ally_x = ally_x_create(hdev); + if (IS_ERR(drvdata.ally_x)) { + hid_err(hdev, "Failed to create Ally X gamepad.\n"); + drvdata.ally_x = NULL; + goto err_close; + } + hid_info(hdev, "Created Ally X controller.\n"); + + // Not required since we send this inputs ep through the gamepad input dev + if (drvdata.gamepad_cfg && drvdata.gamepad_cfg->input) { + input_unregister_device(drvdata.gamepad_cfg->input); + hid_info(hdev, "Ally X removed unrequired input dev.\n"); + } + } + + return 0; + +err_close: + hid_hw_close(hdev); +err_stop: + hid_hw_stop(hdev); + return ret; +} + +static void ally_hid_remove(struct hid_device *hdev) +{ + if (drvdata.led_rgb_dev) + ally_rgb_remove(hdev); + + if (drvdata.ally_x) + ally_x_remove(hdev); + + if (drvdata.gamepad_cfg) + ally_cfg_remove(hdev); + + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static int ally_hid_resume(struct hid_device *hdev) +{ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; + int err; + + if (!ally_cfg) + return 0; + + err = _gamepad_apply_all(hdev, ally_cfg); + if (err) + return err; + + return 0; +} + +static int ally_hid_reset_resume(struct hid_device *hdev) +{ + int ep = get_endpoint_address(hdev); + if (ep != ROG_ALLY_CFG_INTF_IN) + return 0; + + ally_hid_init(hdev); + ally_rgb_resume(); + + return ally_hid_resume(hdev); +} + +static int ally_pm_thaw(struct device *dev) +{ + struct hid_device *hdev = to_hid_device(dev); + + return ally_hid_reset_resume(hdev); +} + +static int ally_pm_suspend(struct device *dev) +{ + if (drvdata.led_rgb_dev) { + ally_rgb_store_settings(); + } + + return 0; +} + +static const struct dev_pm_ops ally_pm_ops = { + .thaw = ally_pm_thaw, + .suspend = ally_pm_suspend, + .poweroff = ally_pm_suspend, +}; + +MODULE_DEVICE_TABLE(hid, rog_ally_devices); + +static struct hid_driver rog_ally_cfg = { .name = "asus_rog_ally", + .id_table = rog_ally_devices, + .probe = ally_hid_probe, + .remove = ally_hid_remove, + .raw_event = ally_raw_event, + /* HID is the better place for resume functions, not pm_ops */ + .resume = ally_hid_resume, + /* ALLy 1 requires this to reset device state correctly */ + .reset_resume = ally_hid_reset_resume, + .driver = { + .pm = &ally_pm_ops, + } +}; + +static int __init rog_ally_init(void) +{ + return hid_register_driver(&rog_ally_cfg); +} + +static void __exit rog_ally_exit(void) +{ + hid_unregister_driver(&rog_ally_cfg); +} + +module_init(rog_ally_init); +module_exit(rog_ally_exit); + +MODULE_IMPORT_NS("ASUS_WMI"); +MODULE_IMPORT_NS("HID_ASUS"); +MODULE_AUTHOR("Luke D. Jones"); +MODULE_DESCRIPTION("HID Driver for ASUS ROG Ally gamepad configuration."); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-asus-ally.h b/drivers/hid/hid-asus-ally.h new file mode 100644 index 000000000000..c83817589082 --- /dev/null +++ b/drivers/hid/hid-asus-ally.h @@ -0,0 +1,398 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2023 Luke Jones + */ + +#include +#include + +/* + * the xpad_mode is used inside the mode setting packet and is used + * for indexing (xpad_mode - 1) + */ +enum xpad_mode { + xpad_mode_game = 0x01, + xpad_mode_wasd = 0x02, + xpad_mode_mouse = 0x03, +}; + +/* the xpad_cmd determines which feature is set or queried */ +enum xpad_cmd { + xpad_cmd_set_mode = 0x01, + xpad_cmd_set_mapping = 0x02, + xpad_cmd_set_js_dz = 0x04, /* deadzones */ + xpad_cmd_set_tr_dz = 0x05, /* deadzones */ + xpad_cmd_set_vibe_intensity = 0x06, + xpad_cmd_set_leds = 0x08, + xpad_cmd_check_ready = 0x0A, + xpad_cmd_set_turbo = 0x0F, + xpad_cmd_set_response_curve = 0x13, + xpad_cmd_set_adz = 0x18, +}; + +/* the xpad_cmd determines which feature is set or queried */ +enum xpad_cmd_len { + xpad_cmd_len_mode = 0x01, + xpad_cmd_len_mapping = 0x2c, + xpad_cmd_len_deadzone = 0x04, + xpad_cmd_len_vibe_intensity = 0x02, + xpad_cmd_len_leds = 0x0C, + xpad_cmd_len_turbo = 0x20, + xpad_cmd_len_response_curve = 0x09, + xpad_cmd_len_adz = 0x02, +}; + +/* Values correspond to the actual HID byte value required */ +enum btn_pair_index { + btn_pair_dpad_u_d = 0x01, + btn_pair_dpad_l_r = 0x02, + btn_pair_ls_rs = 0x03, + btn_pair_lb_rb = 0x04, + btn_pair_a_b = 0x05, + btn_pair_x_y = 0x06, + btn_pair_view_menu = 0x07, + btn_pair_m1_m2 = 0x08, + btn_pair_lt_rt = 0x09, +}; + +#define BTN_PAD_A 0x0101000000000000 +#define BTN_PAD_B 0x0102000000000000 +#define BTN_PAD_X 0x0103000000000000 +#define BTN_PAD_Y 0x0104000000000000 +#define BTN_PAD_LB 0x0105000000000000 +#define BTN_PAD_RB 0x0106000000000000 +#define BTN_PAD_LS 0x0107000000000000 +#define BTN_PAD_RS 0x0108000000000000 +#define BTN_PAD_DPAD_UP 0x0109000000000000 +#define BTN_PAD_DPAD_DOWN 0x010A000000000000 +#define BTN_PAD_DPAD_LEFT 0x010B000000000000 +#define BTN_PAD_DPAD_RIGHT 0x010C000000000000 +#define BTN_PAD_LT 0x010D000000000000 +#define BTN_PAD_RT 0x010E000000000000 +#define BTN_PAD_VIEW 0x0111000000000000 +#define BTN_PAD_MENU 0x0112000000000000 +#define BTN_PAD_XBOX 0x0113000000000000 + +#define BTN_KB_M2 0x02008E0000000000 +#define BTN_KB_M1 0x02008F0000000000 +#define BTN_KB_ESC 0x0200760000000000 +#define BTN_KB_F1 0x0200500000000000 +#define BTN_KB_F2 0x0200600000000000 +#define BTN_KB_F3 0x0200400000000000 +#define BTN_KB_F4 0x02000C0000000000 +#define BTN_KB_F5 0x0200030000000000 +#define BTN_KB_F6 0x02000B0000000000 +#define BTN_KB_F7 0x0200800000000000 +#define BTN_KB_F8 0x02000A0000000000 +#define BTN_KB_F9 0x0200010000000000 +#define BTN_KB_F10 0x0200090000000000 +#define BTN_KB_F11 0x0200780000000000 +#define BTN_KB_F12 0x0200070000000000 +#define BTN_KB_F14 0x0200180000000000 +#define BTN_KB_F15 0x0200100000000000 +#define BTN_KB_BACKTICK 0x02000E0000000000 +#define BTN_KB_1 0x0200160000000000 +#define BTN_KB_2 0x02001E0000000000 +#define BTN_KB_3 0x0200260000000000 +#define BTN_KB_4 0x0200250000000000 +#define BTN_KB_5 0x02002E0000000000 +#define BTN_KB_6 0x0200360000000000 +#define BTN_KB_7 0x02003D0000000000 +#define BTN_KB_8 0x02003E0000000000 +#define BTN_KB_9 0x0200460000000000 +#define BTN_KB_0 0x0200450000000000 +#define BTN_KB_HYPHEN 0x02004E0000000000 +#define BTN_KB_EQUALS 0x0200550000000000 +#define BTN_KB_BACKSPACE 0x0200660000000000 +#define BTN_KB_TAB 0x02000D0000000000 +#define BTN_KB_Q 0x0200150000000000 +#define BTN_KB_W 0x02001D0000000000 +#define BTN_KB_E 0x0200240000000000 +#define BTN_KB_R 0x02002D0000000000 +#define BTN_KB_T 0x02002C0000000000 +#define BTN_KB_Y 0x0200350000000000 +#define BTN_KB_U 0x02003C0000000000 +#define BTN_KB_O 0x0200440000000000 +#define BTN_KB_P 0x02004D0000000000 +#define BTN_KB_LBRACKET 0x0200540000000000 +#define BTN_KB_RBRACKET 0x02005B0000000000 +#define BTN_KB_BACKSLASH 0x02005D0000000000 +#define BTN_KB_CAPS 0x0200580000000000 +#define BTN_KB_A 0x02001C0000000000 +#define BTN_KB_S 0x02001B0000000000 +#define BTN_KB_D 0x0200230000000000 +#define BTN_KB_F 0x02002B0000000000 +#define BTN_KB_G 0x0200340000000000 +#define BTN_KB_H 0x0200330000000000 +#define BTN_KB_J 0x02003B0000000000 +#define BTN_KB_K 0x0200420000000000 +#define BTN_KB_L 0x02004B0000000000 +#define BTN_KB_SEMI 0x02004C0000000000 +#define BTN_KB_QUOTE 0x0200520000000000 +#define BTN_KB_RET 0x02005A0000000000 +#define BTN_KB_LSHIFT 0x0200880000000000 +#define BTN_KB_Z 0x02001A0000000000 +#define BTN_KB_X 0x0200220000000000 +#define BTN_KB_C 0x0200210000000000 +#define BTN_KB_V 0x02002A0000000000 +#define BTN_KB_B 0x0200320000000000 +#define BTN_KB_N 0x0200310000000000 +#define BTN_KB_M 0x02003A0000000000 +#define BTN_KB_COMMA 0x0200410000000000 +#define BTN_KB_PERIOD 0x0200490000000000 +#define BTN_KB_RSHIFT 0x0200890000000000 +#define BTN_KB_LCTL 0x02008C0000000000 +#define BTN_KB_META 0x0200820000000000 +#define BTN_KB_LALT 0x02008A0000000000 +#define BTN_KB_SPACE 0x0200290000000000 +#define BTN_KB_RALT 0x02008B0000000000 +#define BTN_KB_MENU 0x0200840000000000 +#define BTN_KB_RCTL 0x02008D0000000000 +#define BTN_KB_PRNTSCN 0x0200C30000000000 +#define BTN_KB_SCRLCK 0x02007E0000000000 +#define BTN_KB_PAUSE 0x0200910000000000 +#define BTN_KB_INS 0x0200C20000000000 +#define BTN_KB_HOME 0x0200940000000000 +#define BTN_KB_PGUP 0x0200960000000000 +#define BTN_KB_DEL 0x0200C00000000000 +#define BTN_KB_END 0x0200950000000000 +#define BTN_KB_PGDWN 0x0200970000000000 +#define BTN_KB_UP_ARROW 0x0200980000000000 +#define BTN_KB_DOWN_ARROW 0x0200990000000000 +#define BTN_KB_LEFT_ARROW 0x0200910000000000 +#define BTN_KB_RIGHT_ARROW 0x02009B0000000000 + +#define BTN_NUMPAD_LOCK 0x0200770000000000 +#define BTN_NUMPAD_FWDSLASH 0x0200900000000000 +#define BTN_NUMPAD_ASTERISK 0x02007C0000000000 +#define BTN_NUMPAD_HYPHEN 0x02007B0000000000 +#define BTN_NUMPAD_0 0x0200700000000000 +#define BTN_NUMPAD_1 0x0200690000000000 +#define BTN_NUMPAD_2 0x0200720000000000 +#define BTN_NUMPAD_3 0x02007A0000000000 +#define BTN_NUMPAD_4 0x02006B0000000000 +#define BTN_NUMPAD_5 0x0200730000000000 +#define BTN_NUMPAD_6 0x0200740000000000 +#define BTN_NUMPAD_7 0x02006C0000000000 +#define BTN_NUMPAD_8 0x0200750000000000 +#define BTN_NUMPAD_9 0x02007D0000000000 +#define BTN_NUMPAD_PLUS 0x0200790000000000 +#define BTN_NUMPAD_ENTER 0x0200810000000000 +#define BTN_NUMPAD_PERIOD 0x0200710000000000 + +#define BTN_MOUSE_LCLICK 0x0300000001000000 +#define BTN_MOUSE_RCLICK 0x0300000002000000 +#define BTN_MOUSE_MCLICK 0x0300000003000000 +#define BTN_MOUSE_WHEEL_UP 0x0300000004000000 +#define BTN_MOUSE_WHEEL_DOWN 0x0300000005000000 + +#define BTN_MEDIA_SCREENSHOT 0x0500001600000000 +#define BTN_MEDIA_SHOW_KEYBOARD 0x0500001900000000 +#define BTN_MEDIA_SHOW_DESKTOP 0x0500001C00000000 +#define BTN_MEDIA_START_RECORDING 0x0500001E00000000 +#define BTN_MEDIA_MIC_OFF 0x0500000100000000 +#define BTN_MEDIA_VOL_DOWN 0x0500000200000000 +#define BTN_MEDIA_VOL_UP 0x0500000300000000 + +#define ALLY_DEVICE_ATTR_WO(_name, _sysfs_name) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_sysfs_name, 0200, NULL, _name##_store) + +/* required so we can have nested attributes with same name but different functions */ +#define ALLY_DEVICE_ATTR_RW(_name, _sysfs_name) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_sysfs_name, 0644, _name##_show, _name##_store) + +#define ALLY_DEVICE_ATTR_RO(_name, _sysfs_name) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_sysfs_name, 0444, _name##_show, NULL) + +/* button specific macros */ +#define ALLY_BTN_SHOW(_fname, _btn_name, _secondary) \ + static ssize_t _fname##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + struct btn_data *btn; \ + const char* name; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + btn = &ally_cfg->key_mapping[ally_cfg->mode - 1]._btn_name; \ + name = btn_to_name(_secondary ? btn->macro : btn->button); \ + return sysfs_emit(buf, "%s\n", name); \ + } + +#define ALLY_BTN_STORE(_fname, _btn_name, _secondary) \ + static ssize_t _fname##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + struct btn_data *btn; \ + u64 code; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + btn = &ally_cfg->key_mapping[ally_cfg->mode - 1]._btn_name; \ + code = name_to_btn(buf); \ + if (_secondary) \ + btn->macro = code; \ + else \ + btn->button = code; \ + return count; \ + } + +#define ALLY_TURBO_SHOW(_fname, _btn_name) \ + static ssize_t _fname##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + struct btn_data *btn; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + btn = &ally_cfg->key_mapping[ally_cfg->mode - 1]._btn_name; \ + return sysfs_emit(buf, "%d\n", btn->turbo); \ + } + +#define ALLY_TURBO_STORE(_fname, _btn_name) \ + static ssize_t _fname##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + struct btn_data *btn; \ + bool turbo; \ + int ret; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + btn = &ally_cfg->key_mapping[ally_cfg->mode - 1]._btn_name; \ + ret = kstrtobool(buf, &turbo); \ + if (ret) \ + return ret; \ + btn->turbo = turbo; \ + return count; \ + } + +#define ALLY_DEADZONE_SHOW(_fname, _axis_name) \ + static ssize_t _fname##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + struct deadzone *dz; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + dz = &ally_cfg->_axis_name; \ + return sysfs_emit(buf, "%d %d\n", dz->inner, dz->outer); \ + } + +#define ALLY_DEADZONE_STORE(_fname, _axis_name) \ + static ssize_t _fname##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + struct hid_device *hdev = to_hid_device(dev); \ + u32 inner, outer; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + if (sscanf(buf, "%d %d", &inner, &outer) != 2) \ + return -EINVAL; \ + if (inner > 64 || outer > 64 || inner > outer) \ + return -EINVAL; \ + ally_cfg->_axis_name.inner = inner; \ + ally_cfg->_axis_name.outer = outer; \ + _gamepad_apply_deadzones(hdev, ally_cfg); \ + return count; \ + } + +#define ALLY_DEADZONES(_fname, _mname) \ + ALLY_DEADZONE_SHOW(_fname##_deadzone, _mname); \ + ALLY_DEADZONE_STORE(_fname##_deadzone, _mname); \ + ALLY_DEVICE_ATTR_RW(_fname##_deadzone, deadzone) + +/* response curve macros */ +#define ALLY_RESP_CURVE_SHOW(_fname, _mname) \ +static ssize_t _fname##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + return sysfs_emit(buf, "%d\n", ally_cfg->ls_rc._mname); \ + } + +#define ALLY_RESP_CURVE_STORE(_fname, _mname) \ +static ssize_t _fname##_store(struct device *dev, \ + struct device_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct ally_gamepad_cfg *ally_cfg = drvdata.gamepad_cfg; \ + int ret, val; \ + if (!drvdata.gamepad_cfg) \ + return -ENODEV; \ + ret = kstrtoint(buf, 0, &val); \ + if (ret) \ + return ret; \ + if (val < 0 || val > 100) \ + return -EINVAL; \ + ally_cfg->ls_rc._mname = val; \ + return count; \ + } + +/* _point_n must start at 1 */ +#define ALLY_JS_RC_POINT(_fname, _mname, _num) \ + ALLY_RESP_CURVE_SHOW(_fname##_##_mname##_##_num, _mname##_pct_##_num); \ + ALLY_RESP_CURVE_STORE(_fname##_##_mname##_##_num, _mname##_pct_##_num); \ + ALLY_DEVICE_ATTR_RW(_fname##_##_mname##_##_num, curve_##_mname##_pct_##_num) + +#define ALLY_BTN_ATTRS_GROUP(_name, _fname) \ + static struct attribute *_fname##_attrs[] = { \ + &dev_attr_##_fname.attr, \ + &dev_attr_##_fname##_macro.attr, \ + }; \ + static const struct attribute_group _fname##_attr_group = { \ + .name = __stringify(_name), \ + .attrs = _fname##_attrs, \ + } + +#define _ALLY_BTN_REMAP(_fname, _btn_name) \ + ALLY_BTN_SHOW(btn_mapping_##_fname##_remap, _btn_name, false); \ + ALLY_BTN_STORE(btn_mapping_##_fname##_remap, _btn_name, false); \ + ALLY_DEVICE_ATTR_RW(btn_mapping_##_fname##_remap, remap); + +#define _ALLY_BTN_MACRO(_fname, _btn_name) \ + ALLY_BTN_SHOW(btn_mapping_##_fname##_macro, _btn_name, true); \ + ALLY_BTN_STORE(btn_mapping_##_fname##_macro, _btn_name, true); \ + ALLY_DEVICE_ATTR_RW(btn_mapping_##_fname##_macro, macro_remap); + +#define ALLY_BTN_MAPPING(_fname, _btn_name) \ + _ALLY_BTN_REMAP(_fname, _btn_name) \ + _ALLY_BTN_MACRO(_fname, _btn_name) \ + static struct attribute *_fname##_attrs[] = { \ + &dev_attr_btn_mapping_##_fname##_remap.attr, \ + &dev_attr_btn_mapping_##_fname##_macro.attr, \ + NULL, \ + }; \ + static const struct attribute_group btn_mapping_##_fname##_attr_group = { \ + .name = __stringify(btn_##_fname), \ + .attrs = _fname##_attrs, \ + } + +#define ALLY_TURBO_BTN_MAPPING(_fname, _btn_name) \ + _ALLY_BTN_REMAP(_fname, _btn_name) \ + _ALLY_BTN_MACRO(_fname, _btn_name) \ + ALLY_TURBO_SHOW(btn_mapping_##_fname##_turbo, _btn_name); \ + ALLY_TURBO_STORE(btn_mapping_##_fname##_turbo, _btn_name); \ + ALLY_DEVICE_ATTR_RW(btn_mapping_##_fname##_turbo, turbo); \ + static struct attribute *_fname##_turbo_attrs[] = { \ + &dev_attr_btn_mapping_##_fname##_remap.attr, \ + &dev_attr_btn_mapping_##_fname##_macro.attr, \ + &dev_attr_btn_mapping_##_fname##_turbo.attr, \ + NULL, \ + }; \ + static const struct attribute_group btn_mapping_##_fname##_attr_group = { \ + .name = __stringify(btn_##_fname), \ + .attrs = _fname##_turbo_attrs, \ + } diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 4b45e31f0bab..38bc8e2effda 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -23,6 +23,8 @@ /* */ +#include +#include "linux/export.h" #include #include #include @@ -30,9 +32,11 @@ #include #include /* For to_usb_interface for T100 touchpad intf check */ #include +#include #include #include "hid-ids.h" +#include "hid-asus.h" MODULE_AUTHOR("Yusuke Fujimaki "); MODULE_AUTHOR("Brendan McGrath "); @@ -47,10 +51,12 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define T100CHI_MOUSE_REPORT_ID 0x06 #define FEATURE_REPORT_ID 0x0d #define INPUT_REPORT_ID 0x5d +#define HID_USAGE_PAGE_VENDOR 0xff310000 #define FEATURE_KBD_REPORT_ID 0x5a -#define FEATURE_KBD_REPORT_SIZE 16 +#define FEATURE_KBD_REPORT_SIZE 64 #define FEATURE_KBD_LED_REPORT_ID1 0x5d #define FEATURE_KBD_LED_REPORT_ID2 0x5e +#define FEATURE_KBD_LED_REPORT_SIZE 7 #define ROG_ALLY_REPORT_SIZE 64 #define ROG_ALLY_X_MIN_MCU 313 @@ -89,6 +95,9 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define QUIRK_ROG_NKEY_KEYBOARD BIT(11) #define QUIRK_ROG_CLAYMORE_II_KEYBOARD BIT(12) #define QUIRK_ROG_ALLY_XPAD BIT(13) +#define QUIRK_HANDLE_GENERIC BIT(14) +#define QUIRK_ROG_NKEY_RGB BIT(15) +#define QUIRK_ROG_NKEY_LEGACY BIT(16) #define I2C_KEYBOARD_QUIRKS (QUIRK_FIX_NOTEBOOK_REPORT | \ QUIRK_NO_INIT_REPORTS | \ @@ -100,10 +109,16 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define TRKID_SGN ((TRKID_MAX + 1) >> 1) struct asus_kbd_leds { - struct led_classdev cdev; + struct asus_hid_listener listener; + struct led_classdev_mc mc_led; + struct mc_subled subled_info[3]; struct hid_device *hdev; struct work_struct work; unsigned int brightness; + u8 rgb_colors[3]; + bool rgb_init; + bool rgb_set; + bool rgb_registered; spinlock_t lock; bool removed; }; @@ -125,7 +140,6 @@ struct asus_drvdata { struct input_dev *tp_kbd_input; struct asus_kbd_leds *kbd_backlight; const struct asus_touchpad_info *tp; - bool enable_backlight; struct power_supply *battery; struct power_supply_desc battery_desc; int battery_capacity; @@ -316,13 +330,24 @@ static int asus_e1239t_event(struct asus_drvdata *drvdat, u8 *data, int size) static int asus_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { - if ((usage->hid & HID_USAGE_PAGE) == 0xff310000 && + if ((usage->hid & HID_USAGE_PAGE) == HID_USAGE_PAGE_VENDOR && (usage->hid & HID_USAGE) != 0x00 && (usage->hid & HID_USAGE) != 0xff && !usage->type) { hid_warn(hdev, "Unmapped Asus vendor usagepage code 0x%02x\n", usage->hid & HID_USAGE); } + if (usage->type == EV_KEY && value) { + switch (usage->code) { + case KEY_KBDILLUMUP: + return !asus_hid_event(ASUS_EV_BRTUP); + case KEY_KBDILLUMDOWN: + return !asus_hid_event(ASUS_EV_BRTDOWN); + case KEY_KBDILLUMTOGGLE: + return !asus_hid_event(ASUS_EV_BRTTOGGLE); + } + } + return 0; } @@ -331,6 +356,10 @@ static int asus_raw_event(struct hid_device *hdev, { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + /* NOOP on generic HID devices to avoid side effects. */ + if (drvdata->quirks & QUIRK_HANDLE_GENERIC) + return 0; + if (drvdata->battery && data[0] == BATTERY_REPORT_ID) return asus_report_battery(drvdata, data, size); @@ -393,14 +422,37 @@ static int asus_kbd_set_report(struct hid_device *hdev, const u8 *buf, size_t bu static int asus_kbd_init(struct hid_device *hdev, u8 report_id) { - const u8 buf[] = { report_id, 0x41, 0x53, 0x55, 0x53, 0x20, 0x54, - 0x65, 0x63, 0x68, 0x2e, 0x49, 0x6e, 0x63, 0x2e, 0x00 }; + /* + * The handshake is first sent as a set_report, then retrieved + * from a get_report. They should be equal. + */ + const u8 buf[] = { report_id, 0x41, 0x53, 0x55, 0x53, 0x20, + 0x54, 0x65, 0x63, 0x68, 0x2e, 0x49, 0x6e, 0x63, 0x2e, 0x00 }; + u8 *readbuf; int ret; ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); - if (ret < 0) - hid_err(hdev, "Asus failed to send init command: %d\n", ret); + if (ret < 0) { + hid_err(hdev, "Asus failed to send handshake: %d\n", ret); + return ret; + } + + readbuf = kzalloc(FEATURE_KBD_REPORT_SIZE, GFP_KERNEL); + if (!readbuf) + return -ENOMEM; + + ret = hid_hw_raw_request(hdev, report_id, readbuf, + FEATURE_KBD_REPORT_SIZE, HID_FEATURE_REPORT, + HID_REQ_GET_REPORT); + if (ret < 0) { + hid_err(hdev, "Asus failed to receive handshake ack: %d\n", ret); + } else if (memcmp(readbuf, buf, sizeof(buf)) != 0) { + hid_warn(hdev, "Asus handshake returned invalid response: %*ph\n", + FEATURE_KBD_REPORT_SIZE, readbuf); + // Do not return error if handshake is wrong to avoid regressions + } + kfree(readbuf); return ret; } @@ -422,7 +474,7 @@ static int asus_kbd_get_functions(struct hid_device *hdev, if (!readbuf) return -ENOMEM; - ret = hid_hw_raw_request(hdev, FEATURE_KBD_REPORT_ID, readbuf, + ret = hid_hw_raw_request(hdev, report_id, readbuf, FEATURE_KBD_REPORT_SIZE, HID_FEATURE_REPORT, HID_REQ_GET_REPORT); if (ret < 0) { @@ -467,11 +519,8 @@ static void asus_schedule_work(struct asus_kbd_leds *led) spin_unlock_irqrestore(&led->lock, flags); } -static void asus_kbd_backlight_set(struct led_classdev *led_cdev, - enum led_brightness brightness) +static void do_asus_kbd_backlight_set(struct asus_kbd_leds *led, int brightness) { - struct asus_kbd_leds *led = container_of(led_cdev, struct asus_kbd_leds, - cdev); unsigned long flags; spin_lock_irqsave(&led->lock, flags); @@ -481,13 +530,47 @@ static void asus_kbd_backlight_set(struct led_classdev *led_cdev, asus_schedule_work(led); } -static enum led_brightness asus_kbd_backlight_get(struct led_classdev *led_cdev) +static void asus_kbd_listener_set(struct asus_hid_listener *listener, + int brightness) +{ + struct asus_kbd_leds *led = container_of(listener, struct asus_kbd_leds, + listener); + do_asus_kbd_backlight_set(led, brightness); + if (led->rgb_registered) { + led->mc_led.led_cdev.brightness = brightness; + led_classdev_notify_brightness_hw_changed(&led->mc_led.led_cdev, + brightness); + } +} + +static void asus_kbd_brightness_set(struct led_classdev *led_cdev, + enum led_brightness brightness) +{ + struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(led_cdev); + struct asus_kbd_leds *led = container_of(mc_cdev, struct asus_kbd_leds, + mc_led); + unsigned long flags; + + spin_lock_irqsave(&led->lock, flags); + led->rgb_colors[0] = mc_cdev->subled_info[0].intensity; + led->rgb_colors[1] = mc_cdev->subled_info[1].intensity; + led->rgb_colors[2] = mc_cdev->subled_info[2].intensity; + led->rgb_set = true; + spin_unlock_irqrestore(&led->lock, flags); + + do_asus_kbd_backlight_set(led, brightness); +} + +static enum led_brightness asus_kbd_brightness_get(struct led_classdev *led_cdev) { - struct asus_kbd_leds *led = container_of(led_cdev, struct asus_kbd_leds, - cdev); + struct led_classdev_mc *mc_led; + struct asus_kbd_leds *led; enum led_brightness brightness; unsigned long flags; + mc_led = lcdev_to_mccdev(led_cdev); + led = container_of(mc_led, struct asus_kbd_leds, mc_led); + spin_lock_irqsave(&led->lock, flags); brightness = led->brightness; spin_unlock_irqrestore(&led->lock, flags); @@ -495,9 +578,8 @@ static enum led_brightness asus_kbd_backlight_get(struct led_classdev *led_cdev) return brightness; } -static void asus_kbd_backlight_work(struct work_struct *work) +static void asus_kbd_backlight_work(struct asus_kbd_leds *led) { - struct asus_kbd_leds *led = container_of(work, struct asus_kbd_leds, work); u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, 0x00 }; int ret; unsigned long flags; @@ -511,34 +593,6 @@ static void asus_kbd_backlight_work(struct work_struct *work) hid_err(led->hdev, "Asus failed to set keyboard backlight: %d\n", ret); } -/* WMI-based keyboard backlight LED control (via asus-wmi driver) takes - * precedence. We only activate HID-based backlight control when the - * WMI control is not available. - */ -static bool asus_kbd_wmi_led_control_present(struct hid_device *hdev) -{ - struct asus_drvdata *drvdata = hid_get_drvdata(hdev); - u32 value; - int ret; - - if (!IS_ENABLED(CONFIG_ASUS_WMI)) - return false; - - if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD && - dmi_check_system(asus_use_hid_led_dmi_ids)) { - hid_info(hdev, "using HID for asus::kbd_backlight\n"); - return false; - } - - ret = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DSTS, - ASUS_WMI_DEVID_KBD_BACKLIGHT, 0, &value); - hid_dbg(hdev, "WMI backlight check: rc %d value %x", ret, value); - if (ret) - return false; - - return !!(value & ASUS_WMI_DSTS_PRESENCE_BIT); -} - /* * We don't care about any other part of the string except the version section. * Example strings: FGA80100.RC72LA.312_T01, FGA80100.RC71LS.318_T01 @@ -601,7 +655,7 @@ static int mcu_request_version(struct hid_device *hdev) return ret; } -static void validate_mcu_fw_version(struct hid_device *hdev, int idProduct) +void validate_mcu_fw_version(struct hid_device *hdev, int idProduct) { int min_version, version; @@ -629,58 +683,112 @@ static void validate_mcu_fw_version(struct hid_device *hdev, int idProduct) set_ally_mcu_powersave(true); } } +EXPORT_SYMBOL_NS(validate_mcu_fw_version, "HID_ASUS"); + +static void asus_kbd_rgb_work(struct asus_kbd_leds *led) +{ + u8 rgb_buf[][FEATURE_KBD_LED_REPORT_SIZE] = { + { FEATURE_KBD_LED_REPORT_ID1, 0xB3 }, /* set mode */ + { FEATURE_KBD_LED_REPORT_ID1, 0xB5 }, /* apply mode */ + { FEATURE_KBD_LED_REPORT_ID1, 0xB4 }, /* save to mem */ + }; + unsigned long flags; + uint8_t colors[3]; + bool rgb_init, rgb_set; + int ret; + + spin_lock_irqsave(&led->lock, flags); + rgb_init = led->rgb_init; + rgb_set = led->rgb_set; + led->rgb_set = false; + colors[0] = led->rgb_colors[0]; + colors[1] = led->rgb_colors[1]; + colors[2] = led->rgb_colors[2]; + spin_unlock_irqrestore(&led->lock, flags); + + if (!rgb_set) + return; + + if (rgb_init) { + ret = asus_kbd_init(led->hdev, FEATURE_KBD_LED_REPORT_ID1); + if (ret < 0) { + hid_err(led->hdev, "Asus failed to init RGB: %d\n", ret); + return; + } + spin_lock_irqsave(&led->lock, flags); + led->rgb_init = false; + spin_unlock_irqrestore(&led->lock, flags); + } + + /* Protocol is: 54b3 zone (0=all) mode (0=solid) RGB */ + rgb_buf[0][4] = colors[0]; + rgb_buf[0][5] = colors[1]; + rgb_buf[0][6] = colors[2]; + + for (size_t i = 0; i < ARRAY_SIZE(rgb_buf); i++) { + ret = asus_kbd_set_report(led->hdev, rgb_buf[i], sizeof(rgb_buf[i])); + if (ret < 0) { + hid_err(led->hdev, "Asus failed to set RGB: %d\n", ret); + return; + } + } +} + +static void asus_kbd_work(struct work_struct *work) +{ + struct asus_kbd_leds *led = container_of(work, struct asus_kbd_leds, + work); + asus_kbd_backlight_work(led); + asus_kbd_rgb_work(led); +} static int asus_kbd_register_leds(struct hid_device *hdev) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); - struct usb_interface *intf; - struct usb_device *udev; + struct asus_kbd_leds *leds; unsigned char kbd_func; + bool rgb_init = true; int ret; - if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD) { - /* Initialize keyboard */ - ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); - if (ret < 0) - return ret; + ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); + if (ret < 0) + return ret; - /* The LED endpoint is initialised in two HID */ + if (drvdata->quirks & QUIRK_ROG_NKEY_LEGACY) { + /* + * These keyboards might need 0x5d for shortcuts to work. + * As it has been more than 5 years, it is hard to verify. + */ ret = asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID1); if (ret < 0) return ret; - ret = asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID2); - if (ret < 0) - return ret; - - if (dmi_match(DMI_PRODUCT_FAMILY, "ProArt P16")) { - ret = asus_kbd_disable_oobe(hdev); - if (ret < 0) - return ret; - } - - if (drvdata->quirks & QUIRK_ROG_ALLY_XPAD) { - intf = to_usb_interface(hdev->dev.parent); - udev = interface_to_usbdev(intf); - validate_mcu_fw_version(hdev, - le16_to_cpu(udev->descriptor.idProduct)); - } + rgb_init = false; + } - } else { - /* Initialize keyboard */ - ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); - if (ret < 0) - return ret; + /* Get keyboard functions */ + ret = asus_kbd_get_functions(hdev, &kbd_func, FEATURE_KBD_REPORT_ID); + if (ret < 0) + return ret; - /* Get keyboard functions */ - ret = asus_kbd_get_functions(hdev, &kbd_func, FEATURE_KBD_REPORT_ID); + if (dmi_match(DMI_PRODUCT_FAMILY, "ProArt P16")) { + ret = asus_kbd_disable_oobe(hdev); if (ret < 0) return ret; + } - /* Check for backlight support */ - if (!(kbd_func & SUPPORT_KBD_BACKLIGHT)) - return -ENODEV; +#if !IS_REACHABLE(CONFIG_HID_ASUS_ALLY) + if (drvdata->quirks & QUIRK_ROG_ALLY_XPAD) { + struct usb_interface *intf = to_usb_interface(hdev->dev.parent); + struct usb_device *udev = interface_to_usbdev(intf); + validate_mcu_fw_version( + hdev, le16_to_cpu(udev->descriptor.idProduct)); } +#endif + + /* Check for backlight support */ + if (!(kbd_func & SUPPORT_KBD_BACKLIGHT)) + return -ENODEV; drvdata->kbd_backlight = devm_kzalloc(&hdev->dev, sizeof(struct asus_kbd_leds), @@ -688,23 +796,47 @@ static int asus_kbd_register_leds(struct hid_device *hdev) if (!drvdata->kbd_backlight) return -ENOMEM; - drvdata->kbd_backlight->removed = false; - drvdata->kbd_backlight->brightness = 0; - drvdata->kbd_backlight->hdev = hdev; - drvdata->kbd_backlight->cdev.name = "asus::kbd_backlight"; - drvdata->kbd_backlight->cdev.max_brightness = 3; - drvdata->kbd_backlight->cdev.brightness_set = asus_kbd_backlight_set; - drvdata->kbd_backlight->cdev.brightness_get = asus_kbd_backlight_get; - INIT_WORK(&drvdata->kbd_backlight->work, asus_kbd_backlight_work); + leds = drvdata->kbd_backlight; + leds->removed = false; + leds->brightness = ASUS_EV_MAX_BRIGHTNESS; + leds->hdev = hdev; + leds->listener.brightness_set = asus_kbd_listener_set; + + leds->rgb_colors[0] = 0; + leds->rgb_colors[1] = 0; + leds->rgb_colors[2] = 0; + leds->rgb_init = rgb_init; + leds->rgb_set = false; + leds->mc_led.led_cdev.name = devm_kasprintf(&hdev->dev, GFP_KERNEL, + "asus-%s:rgb:peripheral", + strlen(hdev->uniq) ? + hdev->uniq : dev_name(&hdev->dev)); + leds->mc_led.led_cdev.flags = LED_BRIGHT_HW_CHANGED; + leds->mc_led.led_cdev.max_brightness = ASUS_EV_MAX_BRIGHTNESS; + leds->mc_led.led_cdev.brightness_set = asus_kbd_brightness_set; + leds->mc_led.led_cdev.brightness_get = asus_kbd_brightness_get; + leds->mc_led.subled_info = leds->subled_info; + leds->mc_led.num_colors = ARRAY_SIZE(leds->subled_info); + leds->subled_info[0].color_index = LED_COLOR_ID_RED; + leds->subled_info[1].color_index = LED_COLOR_ID_GREEN; + leds->subled_info[2].color_index = LED_COLOR_ID_BLUE; + + INIT_WORK(&drvdata->kbd_backlight->work, asus_kbd_work); spin_lock_init(&drvdata->kbd_backlight->lock); - ret = devm_led_classdev_register(&hdev->dev, &drvdata->kbd_backlight->cdev); - if (ret < 0) { - /* No need to have this still around */ - devm_kfree(&hdev->dev, drvdata->kbd_backlight); + ret = asus_hid_register_listener(&drvdata->kbd_backlight->listener); + /* Asus-wmi might not be accessible so this is not fatal. */ + if (!ret) + hid_warn(hdev, "Asus-wmi brightness listener not registered\n"); + + if (drvdata->quirks & QUIRK_ROG_NKEY_RGB) { + ret = devm_led_classdev_multicolor_register(&hdev->dev, &leds->mc_led); + if (!ret) + leds->rgb_registered = true; + return ret; } - return ret; + return 0; } /* @@ -870,6 +1002,10 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi) struct input_dev *input = hi->input; struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + /* NOOP on generic HID devices to avoid side effects. */ + if (drvdata->quirks & QUIRK_HANDLE_GENERIC) + return 0; + /* T100CHI uses MULTI_INPUT, bind the touchpad to the mouse hid_input */ if (drvdata->quirks & QUIRK_T100CHI && hi->report->id != T100CHI_MOUSE_REPORT_ID) @@ -923,11 +1059,6 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi) drvdata->input = input; - if (drvdata->enable_backlight && - !asus_kbd_wmi_led_control_present(hdev) && - asus_kbd_register_leds(hdev)) - hid_warn(hdev, "Failed to initialize backlight.\n"); - return 0; } @@ -947,6 +1078,10 @@ static int asus_input_mapping(struct hid_device *hdev, return -1; } + /* NOOP on generic HID devices to avoid side effects. */ + if (drvdata->quirks & QUIRK_HANDLE_GENERIC) + return 0; + /* * Ignore a bunch of bogus collections in the T100CHI descriptor. * This avoids a bunch of non-functional hid_input devices getting @@ -997,15 +1132,6 @@ static int asus_input_mapping(struct hid_device *hdev, return -1; } - /* - * Check and enable backlight only on devices with UsagePage == - * 0xff31 to avoid initializing the keyboard firmware multiple - * times on devices with multiple HID descriptors but same - * PID/VID. - */ - if (drvdata->quirks & QUIRK_USE_KBD_BACKLIGHT) - drvdata->enable_backlight = true; - set_bit(EV_REP, hi->input->evbit); return 1; } @@ -1098,7 +1224,7 @@ static int __maybe_unused asus_resume(struct hid_device *hdev) { if (drvdata->kbd_backlight) { const u8 buf[] = { FEATURE_KBD_REPORT_ID, 0xba, 0xc5, 0xc4, - drvdata->kbd_backlight->cdev.brightness }; + drvdata->kbd_backlight->brightness }; ret = asus_kbd_set_report(hdev, buf, sizeof(buf)); if (ret < 0) { hid_err(hdev, "Asus failed to set keyboard backlight: %d\n", ret); @@ -1122,8 +1248,12 @@ static int __maybe_unused asus_reset_resume(struct hid_device *hdev) static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) { - int ret; + struct hid_report_enum *rep_enum; struct asus_drvdata *drvdata; + struct usb_host_endpoint *ep; + struct usb_interface *intf; + struct hid_report *rep; + int ret, is_vendor = 0; drvdata = devm_kzalloc(&hdev->dev, sizeof(*drvdata), GFP_KERNEL); if (drvdata == NULL) { @@ -1135,6 +1265,18 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) drvdata->quirks = id->driver_data; + /* Ignore these endpoints as they are used by hid-asus-ally */ + #if IS_REACHABLE(CONFIG_HID_ASUS_ALLY) + if (drvdata->quirks & QUIRK_ROG_ALLY_XPAD) { + intf = to_usb_interface(hdev->dev.parent); + ep = intf->cur_altsetting->endpoint; + if (ep->desc.bEndpointAddress == ROG_ALLY_X_INTF_IN || + ep->desc.bEndpointAddress == ROG_ALLY_CFG_INTF_IN || + ep->desc.bEndpointAddress == ROG_ALLY_CFG_INTF_OUT) + return -ENODEV; + } + #endif /* IS_REACHABLE(CONFIG_HID_ASUS_ALLY) */ + /* * T90CHI's keyboard dock returns same ID values as T100CHI's dock. * Thus, identify T90CHI dock with product name string. @@ -1207,12 +1349,41 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) return ret; } + /* Check for vendor for RGB init and handle generic devices properly. */ + rep_enum = &hdev->report_enum[HID_INPUT_REPORT]; + list_for_each_entry(rep, &rep_enum->report_list, list) { + if ((rep->application & HID_USAGE_PAGE) == HID_USAGE_PAGE_VENDOR) + is_vendor = true; + } + + /* + * For ROG keyboards, make them hid compliant by + * creating one input per application. For interfaces other than + * the vendor one, disable hid-asus handlers. + */ + if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD) { + if (!is_vendor) + drvdata->quirks |= QUIRK_HANDLE_GENERIC; + hdev->quirks |= HID_QUIRK_INPUT_PER_APP; + } + ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); if (ret) { hid_err(hdev, "Asus hw start failed: %d\n", ret); return ret; } + if (is_vendor && (drvdata->quirks & QUIRK_USE_KBD_BACKLIGHT) && + asus_kbd_register_leds(hdev)) + hid_warn(hdev, "Failed to initialize backlight.\n"); + + /* + * For ROG keyboards, skip rename for consistency and ->input check as + * some devices do not have inputs. + */ + if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD) + return 0; + if (!drvdata->input) { hid_err(hdev, "Asus input not registered\n"); ret = -ENOMEM; @@ -1243,6 +1414,8 @@ static void asus_remove(struct hid_device *hdev) unsigned long flags; if (drvdata->kbd_backlight) { + asus_hid_unregister_listener(&drvdata->kbd_backlight->listener); + spin_lock_irqsave(&drvdata->kbd_backlight->lock, flags); drvdata->kbd_backlight->removed = true; spin_unlock_irqrestore(&drvdata->kbd_backlight->lock, flags); @@ -1263,6 +1436,10 @@ static const __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc, { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + /* NOOP on generic HID devices to avoid side effects. */ + if (drvdata->quirks & QUIRK_HANDLE_GENERIC) + return rdesc; + if (drvdata->quirks & QUIRK_FIX_NOTEBOOK_REPORT && *rsize >= 56 && rdesc[54] == 0x25 && rdesc[55] == 0x65) { hid_info(hdev, "Fixing up Asus notebook report descriptor\n"); @@ -1374,22 +1551,21 @@ static const struct hid_device_id asus_devices[] = { QUIRK_USE_KBD_BACKLIGHT }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_NKEY_LEGACY }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, - { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, - USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_NKEY_LEGACY }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_NKEY_RGB }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD}, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD + | QUIRK_ROG_ALLY_XPAD | QUIRK_ROG_NKEY_RGB }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD }, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD + | QUIRK_ROG_ALLY_XPAD | QUIRK_ROG_NKEY_RGB }, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD), QUIRK_ROG_CLAYMORE_II_KEYBOARD }, @@ -1410,6 +1586,9 @@ static const struct hid_device_id asus_devices[] = { * Note bind to the HID_GROUP_GENERIC group, so that we only bind to the keyboard * part, while letting hid-multitouch.c handle the touchpad. */ + { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, + USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO), + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_NKEY_RGB }, { HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T101HA_KEYBOARD) }, { } diff --git a/drivers/hid/hid-asus.h b/drivers/hid/hid-asus.h new file mode 100644 index 000000000000..f67dd5a3a1bc --- /dev/null +++ b/drivers/hid/hid-asus.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __HID_ASUS_H +#define __HID_ASUS_H + +#include + +#define ROG_ALLY_CFG_INTF_IN 0x83 +#define ROG_ALLY_CFG_INTF_OUT 0x04 +#define ROG_ALLY_X_INTF_IN 0x87 + +void validate_mcu_fw_version(struct hid_device *hdev, int idProduct); + +#endif /* __HID_ASUS_H */ diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 33cc5820f2be..9458af1c2630 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -220,8 +220,9 @@ #define USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD3 0x1822 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD 0x1866 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2 0x19b6 -#define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3 0x1a30 +#define USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO 0x1a30 #define USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR 0x18c6 +#define USB_DEVICE_ID_ASUSTEK_ROG_RAIKIRI_PAD 0x1abb #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY 0x1abe #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X 0x1b4c #define USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD 0x196b diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index e5cbd58a99f3..1236303b1f17 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -267,6 +267,18 @@ config ASUS_WIRELESS If you choose to compile this driver as a module the module will be called asus-wireless. +config ASUS_ARMOURY + tristate "ASUS Armoury driver" + depends on ASUS_WMI + select FW_ATTR_CLASS + help + Say Y here if you have a WMI aware Asus machine and would like to use the + firmware_attributes API to control various settings typically exposed in + the ASUS Armoury Crate application available on Windows. + + To compile this driver as a module, choose M here: the module will + be called asus-armoury. + config ASUS_WMI tristate "ASUS WMI Driver" depends on ACPI_WMI @@ -289,6 +301,17 @@ config ASUS_WMI To compile this driver as a module, choose M here: the module will be called asus-wmi. +config ASUS_WMI_DEPRECATED_ATTRS + bool "BIOS option support in WMI platform (DEPRECATED)" + depends on ASUS_WMI + default y + help + Say Y to expose the configurable BIOS options through the asus-wmi + driver. + + This can be used with or without the asus-armoury driver which + has the same attributes, but more, and better features. + config ASUS_NB_WMI tristate "Asus Notebook WMI Driver" depends on ASUS_WMI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index bea87a85ae75..7e978fa52c2f 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_APPLE_GMUX) += apple-gmux.o # ASUS obj-$(CONFIG_ASUS_LAPTOP) += asus-laptop.o obj-$(CONFIG_ASUS_WIRELESS) += asus-wireless.o +obj-$(CONFIG_ASUS_ARMOURY) += asus-armoury.o obj-$(CONFIG_ASUS_WMI) += asus-wmi.o obj-$(CONFIG_ASUS_NB_WMI) += asus-nb-wmi.o obj-$(CONFIG_ASUS_TF103C_DOCK) += asus-tf103c-dock.o diff --git a/drivers/platform/x86/asus-armoury.c b/drivers/platform/x86/asus-armoury.c new file mode 100644 index 000000000000..a461be936294 --- /dev/null +++ b/drivers/platform/x86/asus-armoury.c @@ -0,0 +1,1174 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Asus Armoury (WMI) attributes driver. + * + * This driver uses the fw_attributes class to expose various WMI functions + * that are present in many gaming and some non-gaming ASUS laptops. + * + * These typically don't fit anywhere else in the sysfs such as under LED class, + * hwmon or others, and are set in Windows using the ASUS Armoury Crate tool. + * + * Copyright(C) 2024 Luke Jones + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asus-armoury.h" +#include "firmware_attributes_class.h" + +#define ASUS_NB_WMI_EVENT_GUID "0B3CBB35-E3C2-45ED-91C2-4C5A6D195D1C" + +#define ASUS_MINI_LED_MODE_MASK 0x03 +/* Standard modes for devices with only on/off */ +#define ASUS_MINI_LED_OFF 0x00 +#define ASUS_MINI_LED_ON 0x01 +/* Like "on" but the effect is more vibrant or brighter */ +#define ASUS_MINI_LED_STRONG_MODE 0x02 +/* New modes for devices with 3 mini-led mode types */ +#define ASUS_MINI_LED_2024_WEAK 0x00 +#define ASUS_MINI_LED_2024_STRONG 0x01 +#define ASUS_MINI_LED_2024_OFF 0x02 + +/* Power tunable attribute name defines */ +#define ATTR_PPT_PL1_SPL "ppt_pl1_spl" +#define ATTR_PPT_PL2_SPPT "ppt_pl2_sppt" +#define ATTR_PPT_PL3_FPPT "ppt_pl3_fppt" +#define ATTR_PPT_APU_SPPT "ppt_apu_sppt" +#define ATTR_PPT_PLATFORM_SPPT "ppt_platform_sppt" +#define ATTR_NV_DYNAMIC_BOOST "nv_dynamic_boost" +#define ATTR_NV_TEMP_TARGET "nv_temp_target" +#define ATTR_NV_BASE_TGP "nv_base_tgp" +#define ATTR_NV_TGP "nv_tgp" + +#define ASUS_POWER_CORE_MASK GENMASK(15, 8) +#define ASUS_PERF_CORE_MASK GENMASK(7, 0) + +enum cpu_core_type { + CPU_CORE_PERF = 0, + CPU_CORE_POWER, +}; + +enum cpu_core_value { + CPU_CORE_DEFAULT = 0, + CPU_CORE_MIN, + CPU_CORE_MAX, + CPU_CORE_CURRENT, +}; + +#define CPU_PERF_CORE_COUNT_MIN 4 +#define CPU_POWR_CORE_COUNT_MIN 0 + +/* Tunables provided by ASUS for gaming laptops */ +struct cpu_cores { + u32 cur_perf_cores; + u32 min_perf_cores; + u32 max_perf_cores; + u32 cur_power_cores; + u32 min_power_cores; + u32 max_power_cores; +}; + +struct rog_tunables { + const struct power_limits *power_limits; + u32 ppt_pl1_spl; // cpu + u32 ppt_pl2_sppt; // cpu + u32 ppt_pl3_fppt; // cpu + u32 ppt_apu_sppt; // plat + u32 ppt_platform_sppt; // plat + + u32 nv_dynamic_boost; + u32 nv_temp_target; + u32 nv_tgp; +}; + +static struct asus_armoury_priv { + struct device *fw_attr_dev; + struct kset *fw_attr_kset; + + struct cpu_cores *cpu_cores; + /* Index 0 for DC, 1 for AC */ + struct rog_tunables *rog_tunables[2]; + u32 mini_led_dev_id; + u32 gpu_mux_dev_id; + /* + * Mutex to prevent big/little core count changes writing to same + * endpoint at the same time. Must lock during attr store. + */ + struct mutex cpu_core_mutex; +} asus_armoury = { + .cpu_core_mutex = __MUTEX_INITIALIZER(asus_armoury.cpu_core_mutex) +}; + +struct fw_attrs_group { + bool pending_reboot; +}; + +static struct fw_attrs_group fw_attrs = { + .pending_reboot = false, +}; + +struct asus_attr_group { + const struct attribute_group *attr_group; + u32 wmi_devid; +}; + +static bool asus_wmi_is_present(u32 dev_id) +{ + u32 retval; + int status; + + status = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DSTS, dev_id, 0, &retval); + pr_debug("%s called (0x%08x), retval: 0x%08x\n", __func__, dev_id, retval); + + return status == 0 && (retval & ASUS_WMI_DSTS_PRESENCE_BIT); +} + +static void asus_set_reboot_and_signal_event(void) +{ + fw_attrs.pending_reboot = true; + kobject_uevent(&asus_armoury.fw_attr_dev->kobj, KOBJ_CHANGE); +} + +static ssize_t pending_reboot_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", fw_attrs.pending_reboot); +} + +static struct kobj_attribute pending_reboot = __ATTR_RO(pending_reboot); + +static bool asus_bios_requires_reboot(struct kobj_attribute *attr) +{ + return !strcmp(attr->attr.name, "gpu_mux_mode") || + !strcmp(attr->attr.name, "cores_performance") || + !strcmp(attr->attr.name, "cores_efficiency") || + !strcmp(attr->attr.name, "panel_hd_mode"); +} + +static int armoury_wmi_set_devstate(struct kobj_attribute *attr, u32 value, u32 wmi_dev) +{ + u32 result; + int err; + + err = asus_wmi_set_devstate(wmi_dev, value, &result); + if (err) { + pr_err("Failed to set %s: %d\n", attr->attr.name, err); + return err; + } + /* + * !1 is usually considered a fail by ASUS, but some WMI methods do use > 1 + * to return a status code or similar. + */ + if (result < 1) { + pr_err("Failed to set %s: (result): 0x%x\n", attr->attr.name, result); + return -EIO; + } + + return 0; +} + +/** + * attr_uint_store() - Send an uint to wmi method, checks if within min/max exclusive. + * @kobj: Pointer to the driver object. + * @attr: Pointer to the attribute calling this function. + * @buf: The buffer to read from, this is parsed to `uint` type. + * @count: Required by sysfs attribute macros, pass in from the callee attr. + * @min: Minimum accepted value. Below this returns -EINVAL. + * @max: Maximum accepted value. Above this returns -EINVAL. + * @store_value: Pointer to where the parsed value should be stored. + * @wmi_dev: The WMI function ID to use. + * + * This function is intended to be generic so it can be called from any "_store" + * attribute which works only with integers. The integer to be sent to the WMI method + * is range checked and an error returned if out of range. + * + * If the value is valid and WMI is success, then the sysfs attribute is notified + * and if asus_bios_requires_reboot() is true then reboot attribute is also notified. + * + * Returns: Either count, or an error. + */ +static ssize_t attr_uint_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, + size_t count, u32 min, u32 max, u32 *store_value, u32 wmi_dev) +{ + u32 value; + int err; + + err = kstrtouint(buf, 10, &value); + if (err) + return err; + + if (value < min || value > max) + return -EINVAL; + + err = armoury_wmi_set_devstate(attr, value, wmi_dev); + if (err) + return err; + + if (store_value != NULL) + *store_value = value; + sysfs_notify(kobj, NULL, attr->attr.name); + + if (asus_bios_requires_reboot(attr)) + asus_set_reboot_and_signal_event(); + + return count; +} + +static ssize_t enum_type_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "enumeration\n"); +} + +static ssize_t int_type_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "integer\n"); +} + +/* Mini-LED mode **************************************************************/ +static ssize_t mini_led_mode_current_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + u32 value; + int err; + + err = asus_wmi_get_devstate_dsts(asus_armoury.mini_led_dev_id, &value); + if (err) + return err; + + value &= ASUS_MINI_LED_MODE_MASK; + + /* + * Remap the mode values to match previous generation mini-LED. The last gen + * WMI 0 == off, while on this version WMI 2 == off (flipped). + */ + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE2) { + switch (value) { + case ASUS_MINI_LED_2024_WEAK: + value = ASUS_MINI_LED_ON; + break; + case ASUS_MINI_LED_2024_STRONG: + value = ASUS_MINI_LED_STRONG_MODE; + break; + case ASUS_MINI_LED_2024_OFF: + value = ASUS_MINI_LED_OFF; + break; + } + } + + return sysfs_emit(buf, "%u\n", value); +} + +static ssize_t mini_led_mode_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + u32 mode; + int err; + + err = kstrtou32(buf, 10, &mode); + if (err) + return err; + + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE && + mode > ASUS_MINI_LED_ON) + return -EINVAL; + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE2 && + mode > ASUS_MINI_LED_STRONG_MODE) + return -EINVAL; + + /* + * Remap the mode values so expected behaviour is the same as the last + * generation of mini-LED with 0 == off, 1 == on. + */ + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE2) { + switch (mode) { + case ASUS_MINI_LED_OFF: + mode = ASUS_MINI_LED_2024_OFF; + break; + case ASUS_MINI_LED_ON: + mode = ASUS_MINI_LED_2024_WEAK; + break; + case ASUS_MINI_LED_STRONG_MODE: + mode = ASUS_MINI_LED_2024_STRONG; + break; + } + } + + err = armoury_wmi_set_devstate(attr, mode, asus_armoury.mini_led_dev_id); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + + return count; +} + +static ssize_t mini_led_mode_possible_values_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + switch (asus_armoury.mini_led_dev_id) { + case ASUS_WMI_DEVID_MINI_LED_MODE: + return sysfs_emit(buf, "0;1\n"); + case ASUS_WMI_DEVID_MINI_LED_MODE2: + return sysfs_emit(buf, "0;1;2\n"); + default: + return -ENODEV; + } +} + +ATTR_GROUP_ENUM_CUSTOM(mini_led_mode, "mini_led_mode", "Set the mini-LED backlight mode"); + +static ssize_t gpu_mux_mode_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int result, err; + u32 optimus; + + err = kstrtou32(buf, 10, &optimus); + if (err) + return err; + + if (optimus > 1) + return -EINVAL; + + if (asus_wmi_is_present(ASUS_WMI_DEVID_DGPU)) { + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_DGPU, &result); + if (err) + return err; + if (result && !optimus) { + pr_warn("Can not switch MUX to dGPU mode when dGPU is disabled: %02X %02X\n", + result, optimus); + return -ENODEV; + } + } + + if (asus_wmi_is_present(ASUS_WMI_DEVID_EGPU)) { + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_EGPU, &result); + if (err) + return err; + if (result && !optimus) { + pr_warn("Can not switch MUX to dGPU mode when eGPU is enabled\n"); + return -EBUSY; + } + } + + err = armoury_wmi_set_devstate(attr, optimus, asus_armoury.gpu_mux_dev_id); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + asus_set_reboot_and_signal_event(); + + return count; +} +WMI_SHOW_INT(gpu_mux_mode_current_value, "%u\n", asus_armoury.gpu_mux_dev_id); +ATTR_GROUP_BOOL_CUSTOM(gpu_mux_mode, "gpu_mux_mode", "Set the GPU display MUX mode"); + +/* + * A user may be required to store the value twice, typical store first, then + * rescan PCI bus to activate power, then store a second time to save correctly. + */ +static ssize_t dgpu_disable_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int result, err; + u32 disable; + + err = kstrtou32(buf, 10, &disable); + if (err) + return err; + + if (disable > 1) + return -EINVAL; + + if (asus_armoury.gpu_mux_dev_id) { + err = asus_wmi_get_devstate_dsts(asus_armoury.gpu_mux_dev_id, &result); + if (err) + return err; + if (!result && disable) { + pr_warn("Can not disable dGPU when the MUX is in dGPU mode\n"); + return -EBUSY; + } + } + + err = armoury_wmi_set_devstate(attr, disable, ASUS_WMI_DEVID_DGPU); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + + return count; +} +WMI_SHOW_INT(dgpu_disable_current_value, "%d\n", ASUS_WMI_DEVID_DGPU); +ATTR_GROUP_BOOL_CUSTOM(dgpu_disable, "dgpu_disable", "Disable the dGPU"); + +/* The ACPI call to enable the eGPU also disables the internal dGPU */ +static ssize_t egpu_enable_current_value_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int result, err; + u32 enable; + + err = kstrtou32(buf, 10, &enable); + if (err) + return err; + + if (enable > 1) + return -EINVAL; + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_EGPU_CONNECTED, &result); + if (err) { + pr_warn("Failed to get eGPU connection status: %d\n", err); + return err; + } + + if (asus_armoury.gpu_mux_dev_id) { + err = asus_wmi_get_devstate_dsts(asus_armoury.gpu_mux_dev_id, &result); + if (err) { + pr_warn("Failed to get GPU MUX status: %d\n", result); + return err; + } + if (!result && enable) { + pr_warn("Can not enable eGPU when the MUX is in dGPU mode\n"); + return -ENODEV; + } + } + + err = armoury_wmi_set_devstate(attr, enable, ASUS_WMI_DEVID_EGPU); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + + return count; +} +WMI_SHOW_INT(egpu_enable_current_value, "%d\n", ASUS_WMI_DEVID_EGPU); +ATTR_GROUP_BOOL_CUSTOM(egpu_enable, "egpu_enable", "Enable the eGPU (also disables dGPU)"); + +/* Device memory available to APU */ + +/* Values map for APU memory: some looks out of order but are actually correct */ +static u32 apu_mem_map[] = { + [0] = 0x000, /* called "AUTO" on the BIOS, is the minimum available */ + [1] = 0x102, + [2] = 0x103, + [3] = 0x104, + [4] = 0x105, + [5] = 0x107, + [6] = 0x108, + [7] = 0x109, + [8] = 0x106, +}; + +static ssize_t apu_mem_current_value_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + int err; + u32 mem; + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_APU_MEM, &mem); + if (err) + return err; + + if ((mem & ASUS_WMI_DSTS_PRESENCE_BIT) == 0) + return -ENODEV; + + mem &= ~ASUS_WMI_DSTS_PRESENCE_BIT; + + /* After 0x000 is set, a read will return 0x100 */ + if (mem == 0x100) + return sysfs_emit(buf, "0\n"); + + for (unsigned int i = 0; i < ARRAY_SIZE(apu_mem_map); i++) { + if (apu_mem_map[i] == mem) + return sysfs_emit(buf, "%u\n", i); + } + + pr_warn("Unrecognised value for APU mem 0x%08x\n", mem); + return sysfs_emit(buf, "%u\n", mem); +} + +static ssize_t apu_mem_current_value_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int result, err; + u32 requested, mem; + + result = kstrtou32(buf, 10, &requested); + if (result) + return result; + + if (requested > ARRAY_SIZE(apu_mem_map)) + return -EINVAL; + + mem = apu_mem_map[requested]; + + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_APU_MEM, mem, &result); + if (err) { + pr_warn("Failed to set apu_mem: %d\n", err); + return err; + } + + pr_info("APU memory changed to %uGB, reboot required\n", requested+1); + sysfs_notify(kobj, NULL, attr->attr.name); + + asus_set_reboot_and_signal_event(); + + return count; +} + +static ssize_t apu_mem_possible_values_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + BUILD_BUG_ON(ARRAY_SIZE(apu_mem_map) != 9); + return sysfs_emit(buf, "0;1;2;3;4;5;6;7;8\n"); +} +ATTR_GROUP_ENUM_CUSTOM(apu_mem, "apu_mem", "Set available system RAM (in GB) for the APU to use"); + +static int init_max_cpu_cores(void) +{ + u32 cores; + int err; + + asus_armoury.cpu_cores = kzalloc(sizeof(struct cpu_cores), GFP_KERNEL); + if (!asus_armoury.cpu_cores) + return -ENOMEM; + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_CORES_MAX, &cores); + if (err) + return err; + + if ((cores & ASUS_WMI_DSTS_PRESENCE_BIT) == 0) { + pr_err("ACPI does not support CPU core count control\n"); + err = -ENODEV; + goto init_max_cpu_cores_err; + } + + asus_armoury.cpu_cores->max_power_cores = FIELD_GET(ASUS_POWER_CORE_MASK, cores); + asus_armoury.cpu_cores->max_perf_cores = FIELD_GET(ASUS_PERF_CORE_MASK, cores); + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_CORES, &cores); + if (err) { + pr_err("Could not get CPU core count: error %d\n", err); + goto init_max_cpu_cores_err; + } + + asus_armoury.cpu_cores->cur_perf_cores = FIELD_GET(ASUS_PERF_CORE_MASK, cores); + asus_armoury.cpu_cores->cur_power_cores = FIELD_GET(ASUS_POWER_CORE_MASK, cores); + + asus_armoury.cpu_cores->min_perf_cores = CPU_PERF_CORE_COUNT_MIN; + asus_armoury.cpu_cores->min_power_cores = CPU_POWR_CORE_COUNT_MIN; + + return 0; + +init_max_cpu_cores_err: + kfree(asus_armoury.cpu_cores); + return err; +} + +static ssize_t cores_value_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf, + enum cpu_core_type core_type, enum cpu_core_value core_value) +{ + u32 cores; + + switch (core_value) { + case CPU_CORE_DEFAULT: + case CPU_CORE_MAX: + if (core_type == CPU_CORE_PERF) + return sysfs_emit(buf, "%u\n", + asus_armoury.cpu_cores->max_perf_cores); + else + return sysfs_emit(buf, "%u\n", + asus_armoury.cpu_cores->max_power_cores); + case CPU_CORE_MIN: + if (core_type == CPU_CORE_PERF) + return sysfs_emit(buf, "%u\n", + asus_armoury.cpu_cores->min_perf_cores); + else + return sysfs_emit(buf, "%u\n", + asus_armoury.cpu_cores->min_power_cores); + default: + break; + } + + if (core_type == CPU_CORE_PERF) + cores = asus_armoury.cpu_cores->cur_perf_cores; + else + cores = asus_armoury.cpu_cores->cur_power_cores; + + return sysfs_emit(buf, "%u\n", cores); +} + +static ssize_t cores_current_value_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, enum cpu_core_type core_type) +{ + u32 new_cores, perf_cores, power_cores, out_val, min, max; + int result, err; + + result = kstrtou32(buf, 10, &new_cores); + if (result) + return result; + + scoped_guard(mutex, &asus_armoury.cpu_core_mutex) { + if (core_type == CPU_CORE_PERF) { + perf_cores = new_cores; + power_cores = asus_armoury.cpu_cores->cur_power_cores; + min = asus_armoury.cpu_cores->min_perf_cores; + max = asus_armoury.cpu_cores->max_perf_cores; + } else { + perf_cores = asus_armoury.cpu_cores->cur_perf_cores; + power_cores = new_cores; + min = asus_armoury.cpu_cores->min_power_cores; + max = asus_armoury.cpu_cores->max_power_cores; + } + + if (new_cores < min || new_cores > max) + return -EINVAL; + + out_val = FIELD_PREP(ASUS_PERF_CORE_MASK, perf_cores) | + FIELD_PREP(ASUS_POWER_CORE_MASK, power_cores); + + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_CORES, out_val, &result); + if (err) { + pr_warn("Failed to set CPU core count: %d\n", err); + return err; + } + + if (result > 1) { + pr_warn("Failed to set CPU core count (result): 0x%x\n", result); + return -EIO; + } + } + + pr_info("CPU core count changed, reboot required\n"); + + sysfs_notify(kobj, NULL, attr->attr.name); + asus_set_reboot_and_signal_event(); + + return 0; +} + +static ssize_t cores_performance_min_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_MIN); +} + +static ssize_t cores_performance_max_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_MAX); +} + +static ssize_t cores_performance_default_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_DEFAULT); +} + +static ssize_t cores_performance_current_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_CURRENT); +} + +static ssize_t cores_performance_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int err; + + err = cores_current_value_store(kobj, attr, buf, CPU_CORE_PERF); + if (err) + return err; + + return count; +} +ATTR_GROUP_CORES_RW(cores_performance, "cores_performance", + "Set the max available performance cores"); + +static ssize_t cores_efficiency_min_value_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_MIN); +} + +static ssize_t cores_efficiency_max_value_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_MAX); +} + +static ssize_t cores_efficiency_default_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_DEFAULT); +} + +static ssize_t cores_efficiency_current_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_CURRENT); +} + +static ssize_t cores_efficiency_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int err; + + err = cores_current_value_store(kobj, attr, buf, CPU_CORE_POWER); + if (err) + return err; + + return count; +} +ATTR_GROUP_CORES_RW(cores_efficiency, "cores_efficiency", + "Set the max available efficiency cores"); + +/* Define helper to access the current power mode tunable values */ +static inline struct rog_tunables *get_current_tunables(void) +{ + return asus_armoury + .rog_tunables[power_supply_is_system_supplied() ? 1 : 0]; +} + +/* Simple attribute creation */ +ATTR_GROUP_ROG_TUNABLE(ppt_pl1_spl, ATTR_PPT_PL1_SPL, ASUS_WMI_DEVID_PPT_PL1_SPL, + "Set the CPU slow package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_pl2_sppt, ATTR_PPT_PL2_SPPT, ASUS_WMI_DEVID_PPT_PL2_SPPT, + "Set the CPU fast package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_pl3_fppt, ATTR_PPT_PL3_FPPT, ASUS_WMI_DEVID_PPT_FPPT, + "Set the CPU fastest package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_apu_sppt, ATTR_PPT_APU_SPPT, ASUS_WMI_DEVID_PPT_APU_SPPT, + "Set the APU package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_platform_sppt, ATTR_PPT_PLATFORM_SPPT, ASUS_WMI_DEVID_PPT_PLAT_SPPT, + "Set the platform package limit"); +ATTR_GROUP_ROG_TUNABLE(nv_dynamic_boost, ATTR_NV_DYNAMIC_BOOST, ASUS_WMI_DEVID_NV_DYN_BOOST, + "Set the Nvidia dynamic boost limit"); +ATTR_GROUP_ROG_TUNABLE(nv_temp_target, ATTR_NV_TEMP_TARGET, ASUS_WMI_DEVID_NV_THERM_TARGET, + "Set the Nvidia max thermal limit"); +ATTR_GROUP_ROG_TUNABLE(nv_tgp, "nv_tgp", ASUS_WMI_DEVID_DGPU_SET_TGP, + "Set the additional TGP on top of the base TGP"); +ATTR_GROUP_INT_VALUE_ONLY_RO(nv_base_tgp, ATTR_NV_BASE_TGP, ASUS_WMI_DEVID_DGPU_BASE_TGP, + "Read the base TGP value"); + + +ATTR_GROUP_ENUM_INT_RO(charge_mode, "charge_mode", ASUS_WMI_DEVID_CHARGE_MODE, "0;1;2", + "Show the current mode of charging"); + +ATTR_GROUP_BOOL_RW(boot_sound, "boot_sound", ASUS_WMI_DEVID_BOOT_SOUND, + "Set the boot POST sound"); +ATTR_GROUP_BOOL_RW(mcu_powersave, "mcu_powersave", ASUS_WMI_DEVID_MCU_POWERSAVE, + "Set MCU powersaving mode"); +ATTR_GROUP_BOOL_RW(panel_od, "panel_overdrive", ASUS_WMI_DEVID_PANEL_OD, + "Set the panel refresh overdrive"); +ATTR_GROUP_BOOL_RW(panel_hd_mode, "panel_hd_mode", ASUS_WMI_DEVID_PANEL_HD, + "Set the panel HD mode to UHD<0> or FHD<1>"); +ATTR_GROUP_BOOL_RW(screen_auto_brightness, "screen_auto_brightness", + ASUS_WMI_DEVID_SCREEN_AUTO_BRIGHTNESS, + "Set the panel brightness to Off<0> or On<1>"); +ATTR_GROUP_BOOL_RO(egpu_connected, "egpu_connected", ASUS_WMI_DEVID_EGPU_CONNECTED, + "Show the eGPU connection status"); + +/* If an attribute does not require any special case handling add it here */ +static const struct asus_attr_group armoury_attr_groups[] = { + { &egpu_connected_attr_group, ASUS_WMI_DEVID_EGPU_CONNECTED }, + { &egpu_enable_attr_group, ASUS_WMI_DEVID_EGPU }, + { &dgpu_disable_attr_group, ASUS_WMI_DEVID_DGPU }, + { &apu_mem_attr_group, ASUS_WMI_DEVID_APU_MEM }, + { &cores_efficiency_attr_group, ASUS_WMI_DEVID_CORES_MAX }, + { &cores_performance_attr_group, ASUS_WMI_DEVID_CORES_MAX }, + + { &ppt_pl1_spl_attr_group, ASUS_WMI_DEVID_PPT_PL1_SPL }, + { &ppt_pl2_sppt_attr_group, ASUS_WMI_DEVID_PPT_PL2_SPPT }, + { &ppt_pl3_fppt_attr_group, ASUS_WMI_DEVID_PPT_FPPT }, + { &ppt_apu_sppt_attr_group, ASUS_WMI_DEVID_PPT_APU_SPPT }, + { &ppt_platform_sppt_attr_group, ASUS_WMI_DEVID_PPT_PLAT_SPPT }, + { &nv_dynamic_boost_attr_group, ASUS_WMI_DEVID_NV_DYN_BOOST }, + { &nv_temp_target_attr_group, ASUS_WMI_DEVID_NV_THERM_TARGET }, + { &nv_base_tgp_attr_group, ASUS_WMI_DEVID_DGPU_BASE_TGP }, + { &nv_tgp_attr_group, ASUS_WMI_DEVID_DGPU_SET_TGP }, + + { &charge_mode_attr_group, ASUS_WMI_DEVID_CHARGE_MODE }, + { &boot_sound_attr_group, ASUS_WMI_DEVID_BOOT_SOUND }, + { &mcu_powersave_attr_group, ASUS_WMI_DEVID_MCU_POWERSAVE }, + { &panel_od_attr_group, ASUS_WMI_DEVID_PANEL_OD }, + { &panel_hd_mode_attr_group, ASUS_WMI_DEVID_PANEL_HD }, + { &screen_auto_brightness_attr_group, ASUS_WMI_DEVID_SCREEN_AUTO_BRIGHTNESS }, +}; + +/** + * is_power_tunable_attr - Determines if an attribute is a power-related tunable + * @name: The name of the attribute to check + * + * This function checks if the given attribute name is related to power tuning. + * + * Return: true if the attribute is a power-related tunable, false otherwise + */ +static bool is_power_tunable_attr(const char *name) +{ + static const char * const power_tunable_attrs[] = { + ATTR_PPT_PL1_SPL, ATTR_PPT_PL2_SPPT, + ATTR_PPT_PL3_FPPT, ATTR_PPT_APU_SPPT, + ATTR_PPT_PLATFORM_SPPT, ATTR_NV_DYNAMIC_BOOST, + ATTR_NV_TEMP_TARGET, ATTR_NV_BASE_TGP, + ATTR_NV_TGP + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(power_tunable_attrs); i++) { + if (!strcmp(name, power_tunable_attrs[i])) + return true; + } + + return false; +} + +/** + * has_valid_limit - Checks if a power-related attribute has a valid limit value + * @name: The name of the attribute to check + * @limits: Pointer to the power_limits structure containing limit values + * + * This function checks if a power-related attribute has a valid limit value. + * It returns false if limits is NULL or if the corresponding limit value is zero. + * + * Return: true if the attribute has a valid limit value, false otherwise + */ +static bool has_valid_limit(const char *name, const struct power_limits *limits) +{ + u32 limit_value = 0; + + if (!limits) + return false; + + if (!strcmp(name, ATTR_PPT_PL1_SPL)) + limit_value = limits->ppt_pl1_spl_max; + else if (!strcmp(name, ATTR_PPT_PL2_SPPT)) + limit_value = limits->ppt_pl2_sppt_max; + else if (!strcmp(name, ATTR_PPT_PL3_FPPT)) + limit_value = limits->ppt_pl3_fppt_max; + else if (!strcmp(name, ATTR_PPT_APU_SPPT)) + limit_value = limits->ppt_apu_sppt_max; + else if (!strcmp(name, ATTR_PPT_PLATFORM_SPPT)) + limit_value = limits->ppt_platform_sppt_max; + else if (!strcmp(name, ATTR_NV_DYNAMIC_BOOST)) + limit_value = limits->nv_dynamic_boost_max; + else if (!strcmp(name, ATTR_NV_TEMP_TARGET)) + limit_value = limits->nv_temp_target_max; + else if (!strcmp(name, ATTR_NV_BASE_TGP) || + !strcmp(name, ATTR_NV_TGP)) + limit_value = limits->nv_tgp_max; + + return limit_value > 0; +} + +static int asus_fw_attr_add(void) +{ + const struct power_limits *limits; + bool should_create; + const char *name; + int err, i; + + asus_armoury.fw_attr_dev = device_create(&firmware_attributes_class, NULL, MKDEV(0, 0), + NULL, "%s", DRIVER_NAME); + if (IS_ERR(asus_armoury.fw_attr_dev)) { + err = PTR_ERR(asus_armoury.fw_attr_dev); + goto fail_class_get; + } + + asus_armoury.fw_attr_kset = kset_create_and_add("attributes", NULL, + &asus_armoury.fw_attr_dev->kobj); + if (!asus_armoury.fw_attr_kset) { + err = -ENOMEM; + goto err_destroy_classdev; + } + + err = sysfs_create_file(&asus_armoury.fw_attr_kset->kobj, &pending_reboot.attr); + if (err) { + pr_err("Failed to create sysfs level attributes\n"); + goto err_destroy_kset; + } + + asus_armoury.mini_led_dev_id = 0; + if (asus_wmi_is_present(ASUS_WMI_DEVID_MINI_LED_MODE)) + asus_armoury.mini_led_dev_id = ASUS_WMI_DEVID_MINI_LED_MODE; + else if (asus_wmi_is_present(ASUS_WMI_DEVID_MINI_LED_MODE2)) + asus_armoury.mini_led_dev_id = ASUS_WMI_DEVID_MINI_LED_MODE2; + + if (asus_armoury.mini_led_dev_id) { + err = sysfs_create_group(&asus_armoury.fw_attr_kset->kobj, + &mini_led_mode_attr_group); + if (err) { + pr_err("Failed to create sysfs-group for mini_led\n"); + goto err_remove_file; + } + } + + asus_armoury.gpu_mux_dev_id = 0; + if (asus_wmi_is_present(ASUS_WMI_DEVID_GPU_MUX)) + asus_armoury.gpu_mux_dev_id = ASUS_WMI_DEVID_GPU_MUX; + else if (asus_wmi_is_present(ASUS_WMI_DEVID_GPU_MUX_VIVO)) + asus_armoury.gpu_mux_dev_id = ASUS_WMI_DEVID_GPU_MUX_VIVO; + + if (asus_armoury.gpu_mux_dev_id) { + err = sysfs_create_group(&asus_armoury.fw_attr_kset->kobj, + &gpu_mux_mode_attr_group); + if (err) { + pr_err("Failed to create sysfs-group for gpu_mux\n"); + goto err_remove_mini_led_group; + } + } + + for (i = 0; i < ARRAY_SIZE(armoury_attr_groups); i++) { + if (!asus_wmi_is_present(armoury_attr_groups[i].wmi_devid)) + continue; + + /* Always create by default, unless PPT is not present */ + should_create = true; + name = armoury_attr_groups[i].attr_group->name; + + /* Check if this is a power-related tunable requiring limits */ + if (asus_armoury.rog_tunables[1] && asus_armoury.rog_tunables[1]->power_limits && + is_power_tunable_attr(name)) { + limits = asus_armoury.rog_tunables[1]->power_limits; + /* Check only AC, if DC is not present then AC won't be either */ + should_create = has_valid_limit(name, limits); + if (!should_create) { + pr_debug("Missing max value on %s for tunable: %s\n", + dmi_get_system_info(DMI_BOARD_NAME), name); + } + } + + if (should_create) { + err = sysfs_create_group(&asus_armoury.fw_attr_kset->kobj, + armoury_attr_groups[i].attr_group); + if (err) { + pr_err("Failed to create sysfs-group for %s\n", + armoury_attr_groups[i].attr_group->name); + goto err_remove_groups; + } + } + } + + return 0; + +err_remove_groups: + while (i--) { + if (asus_wmi_is_present(armoury_attr_groups[i].wmi_devid)) + sysfs_remove_group(&asus_armoury.fw_attr_kset->kobj, + armoury_attr_groups[i].attr_group); + } + if (asus_armoury.gpu_mux_dev_id) + sysfs_remove_group(&asus_armoury.fw_attr_kset->kobj, &gpu_mux_mode_attr_group); +err_remove_mini_led_group: + if (asus_armoury.mini_led_dev_id) + sysfs_remove_group(&asus_armoury.fw_attr_kset->kobj, &mini_led_mode_attr_group); +err_remove_file: + sysfs_remove_file(&asus_armoury.fw_attr_kset->kobj, &pending_reboot.attr); +err_destroy_kset: + kset_unregister(asus_armoury.fw_attr_kset); +err_destroy_classdev: +fail_class_get: + device_destroy(&firmware_attributes_class, MKDEV(0, 0)); + return err; +} + +/* Init / exit ****************************************************************/ + +/* Set up the min/max and defaults for ROG tunables */ +static void init_rog_tunables(void) +{ + const struct power_limits *ac_limits, *dc_limits; + const struct power_data *power_data; + const struct dmi_system_id *dmi_id; + bool ac_initialized = false, dc_initialized = false; + + /* Match the system against the power_limits table */ + dmi_id = dmi_first_match(power_limits); + if (!dmi_id) { + pr_warn("No matching power limits found for this system\n"); + return; + } + + /* Get the power data for this system */ + power_data = dmi_id->driver_data; + if (!power_data) { + pr_info("No power data available for this system\n"); + return; + } + + /* Initialize AC power tunables */ + ac_limits = power_data->ac_data; + if (ac_limits) { + asus_armoury.rog_tunables[1] = + kzalloc(sizeof(*asus_armoury.rog_tunables[1]), GFP_KERNEL); + if (!asus_armoury.rog_tunables[1]) + goto err_nomem; + + asus_armoury.rog_tunables[1]->power_limits = ac_limits; + + /* Set initial AC values */ + asus_armoury.rog_tunables[1]->ppt_pl1_spl = + ac_limits->ppt_pl1_spl_def ? + ac_limits->ppt_pl1_spl_def : + ac_limits->ppt_pl1_spl_max; + + asus_armoury.rog_tunables[1]->ppt_pl2_sppt = + ac_limits->ppt_pl2_sppt_def ? + ac_limits->ppt_pl2_sppt_def : + ac_limits->ppt_pl2_sppt_max; + + asus_armoury.rog_tunables[1]->ppt_pl3_fppt = + ac_limits->ppt_pl3_fppt_def ? + ac_limits->ppt_pl3_fppt_def : + ac_limits->ppt_pl3_fppt_max; + + asus_armoury.rog_tunables[1]->ppt_apu_sppt = + ac_limits->ppt_apu_sppt_def ? + ac_limits->ppt_apu_sppt_def : + ac_limits->ppt_apu_sppt_max; + + asus_armoury.rog_tunables[1]->ppt_platform_sppt = + ac_limits->ppt_platform_sppt_def ? + ac_limits->ppt_platform_sppt_def : + ac_limits->ppt_platform_sppt_max; + + asus_armoury.rog_tunables[1]->nv_dynamic_boost = + ac_limits->nv_dynamic_boost_max; + asus_armoury.rog_tunables[1]->nv_temp_target = + ac_limits->nv_temp_target_max; + asus_armoury.rog_tunables[1]->nv_tgp = ac_limits->nv_tgp_max; + + ac_initialized = true; + pr_debug("AC power limits initialized for %s\n", dmi_id->matches[0].substr); + } + + /* Initialize DC power tunables */ + dc_limits = power_data->dc_data; + if (dc_limits) { + asus_armoury.rog_tunables[0] = + kzalloc(sizeof(*asus_armoury.rog_tunables[0]), GFP_KERNEL); + if (!asus_armoury.rog_tunables[0]) { + if (ac_initialized) + kfree(asus_armoury.rog_tunables[1]); + goto err_nomem; + } + + asus_armoury.rog_tunables[0]->power_limits = dc_limits; + + /* Set initial DC values */ + asus_armoury.rog_tunables[0]->ppt_pl1_spl = + dc_limits->ppt_pl1_spl_def ? + dc_limits->ppt_pl1_spl_def : + dc_limits->ppt_pl1_spl_max; + + asus_armoury.rog_tunables[0]->ppt_pl2_sppt = + dc_limits->ppt_pl2_sppt_def ? + dc_limits->ppt_pl2_sppt_def : + dc_limits->ppt_pl2_sppt_max; + + asus_armoury.rog_tunables[0]->ppt_pl3_fppt = + dc_limits->ppt_pl3_fppt_def ? + dc_limits->ppt_pl3_fppt_def : + dc_limits->ppt_pl3_fppt_max; + + asus_armoury.rog_tunables[0]->ppt_apu_sppt = + dc_limits->ppt_apu_sppt_def ? + dc_limits->ppt_apu_sppt_def : + dc_limits->ppt_apu_sppt_max; + + asus_armoury.rog_tunables[0]->ppt_platform_sppt = + dc_limits->ppt_platform_sppt_def ? + dc_limits->ppt_platform_sppt_def : + dc_limits->ppt_platform_sppt_max; + + asus_armoury.rog_tunables[0]->nv_dynamic_boost = + dc_limits->nv_dynamic_boost_max; + asus_armoury.rog_tunables[0]->nv_temp_target = + dc_limits->nv_temp_target_max; + asus_armoury.rog_tunables[0]->nv_tgp = dc_limits->nv_tgp_max; + + dc_initialized = true; + pr_debug("DC power limits initialized for %s\n", dmi_id->matches[0].substr); + } + + if (!ac_initialized) + pr_debug("No AC PPT limits defined\n"); + + if (!dc_initialized) + pr_debug("No DC PPT limits defined\n"); + + return; + +err_nomem: + pr_err("Failed to allocate memory for tunables\n"); +} + +static int __init asus_fw_init(void) +{ + char *wmi_uid; + int err; + + wmi_uid = wmi_get_acpi_device_uid(ASUS_WMI_MGMT_GUID); + if (!wmi_uid) + return -ENODEV; + + /* + * if equal to "ASUSWMI" then it's DCTS that can't be used for this + * driver, DSTS is required. + */ + if (!strcmp(wmi_uid, ASUS_ACPI_UID_ASUSWMI)) + return -ENODEV; + + if (asus_wmi_is_present(ASUS_WMI_DEVID_CORES_MAX)) { + err = init_max_cpu_cores(); + if (err) { + pr_err("Could not initialise CPU core control %d\n", err); + return err; + } + } + + init_rog_tunables(); + + /* Must always be last step to ensure data is available */ + return asus_fw_attr_add(); +} + +static void __exit asus_fw_exit(void) +{ + sysfs_remove_file(&asus_armoury.fw_attr_kset->kobj, &pending_reboot.attr); + kset_unregister(asus_armoury.fw_attr_kset); + device_destroy(&firmware_attributes_class, MKDEV(0, 0)); + + kfree(asus_armoury.rog_tunables[0]); + kfree(asus_armoury.rog_tunables[1]); +} + +module_init(asus_fw_init); +module_exit(asus_fw_exit); + +MODULE_IMPORT_NS("ASUS_WMI"); +MODULE_AUTHOR("Luke Jones "); +MODULE_DESCRIPTION("ASUS BIOS Configuration Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("wmi:" ASUS_NB_WMI_EVENT_GUID); diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h new file mode 100644 index 000000000000..438768ea14cc --- /dev/null +++ b/drivers/platform/x86/asus-armoury.h @@ -0,0 +1,1278 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Definitions for kernel modules using asus-armoury driver + * + * Copyright (c) 2024 Luke Jones + */ + +#ifndef _ASUS_ARMOURY_H_ +#define _ASUS_ARMOURY_H_ + +#include +#include +#include + +#define DRIVER_NAME "asus-armoury" + +#define __ASUS_ATTR_RO(_func, _name) \ + { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = _func##_##_name##_show, \ + } + +#define __ASUS_ATTR_RO_AS(_name, _show) \ + { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = _show, \ + } + +#define __ASUS_ATTR_RW(_func, _name) \ + __ATTR(_name, 0644, _func##_##_name##_show, _func##_##_name##_store) + +#define __WMI_STORE_INT(_attr, _min, _max, _wmi) \ + static ssize_t _attr##_store(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + return attr_uint_store(kobj, attr, buf, count, _min, \ + _max, NULL, _wmi); \ + } + +#define WMI_SHOW_INT(_attr, _fmt, _wmi) \ + static ssize_t _attr##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ + { \ + u32 result; \ + int err; \ + \ + err = asus_wmi_get_devstate_dsts(_wmi, &result); \ + if (err) \ + return err; \ + return sysfs_emit(buf, _fmt, \ + result & ~ASUS_WMI_DSTS_PRESENCE_BIT); \ + } + +/* Create functions and attributes for use in other macros or on their own */ + +/* Shows a formatted static variable */ +#define __ATTR_SHOW_FMT(_prop, _attrname, _fmt, _val) \ + static ssize_t _attrname##_##_prop##_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + return sysfs_emit(buf, _fmt, _val); \ + } \ + static struct kobj_attribute attr_##_attrname##_##_prop = \ + __ASUS_ATTR_RO(_attrname, _prop) + +#define __ATTR_RO_INT_GROUP_ENUM(_attrname, _wmi, _fsname, _possible, _dispname)\ + WMI_SHOW_INT(_attrname##_current_value, "%d\n", _wmi); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RO(_attrname, current_value); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + __ATTR_SHOW_FMT(possible_values, _attrname, "%s\n", _possible); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +#define __ATTR_RW_INT_GROUP_ENUM(_attrname, _minv, _maxv, _wmi, _fsname,\ + _possible, _dispname) \ + __WMI_STORE_INT(_attrname##_current_value, _minv, _maxv, _wmi); \ + WMI_SHOW_INT(_attrname##_current_value, "%d\n", _wmi); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + __ATTR_SHOW_FMT(possible_values, _attrname, "%s\n", _possible); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* Boolean style enumeration, base macro. Requires adding show/store */ +#define __ATTR_GROUP_ENUM(_attrname, _fsname, _possible, _dispname) \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + __ATTR_SHOW_FMT(possible_values, _attrname, "%s\n", _possible); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +#define ATTR_GROUP_BOOL_RO(_attrname, _fsname, _wmi, _dispname) \ + __ATTR_RO_INT_GROUP_ENUM(_attrname, _wmi, _fsname, "0;1", _dispname) + + +#define ATTR_GROUP_BOOL_RW(_attrname, _fsname, _wmi, _dispname) \ + __ATTR_RW_INT_GROUP_ENUM(_attrname, 0, 1, _wmi, _fsname, "0;1", _dispname) + +#define ATTR_GROUP_ENUM_INT_RO(_attrname, _fsname, _wmi, _possible, _dispname) \ + __ATTR_RO_INT_GROUP_ENUM(_attrname, _wmi, _fsname, _possible, _dispname) + +/* + * Requires _current_value_show(), _current_value_show() + */ +#define ATTR_GROUP_BOOL_CUSTOM(_attrname, _fsname, _dispname) \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + __ATTR_GROUP_ENUM(_attrname, _fsname, "0;1", _dispname) + +/* + * Requires _current_value_show(), _current_value_show() + * and _possible_values_show() + */ +#define ATTR_GROUP_ENUM_CUSTOM(_attrname, _fsname, _dispname) \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + static struct kobj_attribute attr_##_attrname##_possible_values = \ + __ASUS_ATTR_RO(_attrname, possible_values); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* CPU core attributes need a little different in setup */ +#define ATTR_GROUP_CORES_RW(_attrname, _fsname, _dispname) \ + __ATTR_SHOW_FMT(scalar_increment, _attrname, "%d\n", 1); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + static struct kobj_attribute attr_##_attrname##_default_value = \ + __ASUS_ATTR_RO(_attrname, default_value); \ + static struct kobj_attribute attr_##_attrname##_min_value = \ + __ASUS_ATTR_RO(_attrname, min_value); \ + static struct kobj_attribute attr_##_attrname##_max_value = \ + __ASUS_ATTR_RO(_attrname, max_value); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, int_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_default_value.attr, \ + &attr_##_attrname##_min_value.attr, \ + &attr_##_attrname##_max_value.attr, \ + &attr_##_attrname##_scalar_increment.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +#define ATTR_GROUP_INT_VALUE_ONLY_RO(_attrname, _fsname, _wmi, _dispname) \ + WMI_SHOW_INT(_attrname##_current_value, "%d\n", _wmi); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RO(_attrname, current_value); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, int_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_type.attr, NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* + * ROG PPT attributes need a little different in setup as they + * require rog_tunables members. + */ + +#define __ROG_TUNABLE_SHOW(_prop, _attrname, _val) \ + static ssize_t _attrname##_##_prop##_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables || !tunables->power_limits) \ + return -ENODEV; \ + \ + return sysfs_emit(buf, "%d\n", tunables->power_limits->_val); \ + } \ + static struct kobj_attribute attr_##_attrname##_##_prop = \ + __ASUS_ATTR_RO(_attrname, _prop) + +#define __ROG_TUNABLE_SHOW_DEFAULT(_attrname) \ + static ssize_t _attrname##_default_value_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables || !tunables->power_limits) \ + return -ENODEV; \ + \ + return sysfs_emit( \ + buf, "%d\n", \ + tunables->power_limits->_attrname##_def ? \ + tunables->power_limits->_attrname##_def : \ + tunables->power_limits->_attrname##_max); \ + } \ + static struct kobj_attribute attr_##_attrname##_default_value = \ + __ASUS_ATTR_RO(_attrname, default_value) + +#define __ROG_TUNABLE_RW(_attr, _wmi) \ + static ssize_t _attr##_current_value_store( \ + struct kobject *kobj, struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables || !tunables->power_limits) \ + return -ENODEV; \ + \ + return attr_uint_store(kobj, attr, buf, count, \ + tunables->power_limits->_attr##_min, \ + tunables->power_limits->_attr##_max, \ + &tunables->_attr, _wmi); \ + } \ + static ssize_t _attr##_current_value_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables) \ + return -ENODEV; \ + \ + return sysfs_emit(buf, "%u\n", tunables->_attr); \ + } \ + static struct kobj_attribute attr_##_attr##_current_value = \ + __ASUS_ATTR_RW(_attr, current_value) + +#define ATTR_GROUP_ROG_TUNABLE(_attrname, _fsname, _wmi, _dispname) \ + __ROG_TUNABLE_RW(_attrname, _wmi); \ + __ROG_TUNABLE_SHOW_DEFAULT(_attrname); \ + __ROG_TUNABLE_SHOW(min_value, _attrname, _attrname##_min); \ + __ROG_TUNABLE_SHOW(max_value, _attrname, _attrname##_max); \ + __ATTR_SHOW_FMT(scalar_increment, _attrname, "%d\n", 1); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, int_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_default_value.attr, \ + &attr_##_attrname##_min_value.attr, \ + &attr_##_attrname##_max_value.attr, \ + &attr_##_attrname##_scalar_increment.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* Default is always the maximum value unless *_def is specified */ +struct power_limits { + u8 ppt_pl1_spl_min; + u8 ppt_pl1_spl_def; + u8 ppt_pl1_spl_max; + u8 ppt_pl2_sppt_min; + u8 ppt_pl2_sppt_def; + u8 ppt_pl2_sppt_max; + u8 ppt_pl3_fppt_min; + u8 ppt_pl3_fppt_def; + u8 ppt_pl3_fppt_max; + u8 ppt_apu_sppt_min; + u8 ppt_apu_sppt_def; + u8 ppt_apu_sppt_max; + u8 ppt_platform_sppt_min; + u8 ppt_platform_sppt_def; + u8 ppt_platform_sppt_max; + /* Nvidia GPU specific, default is always max */ + u8 nv_dynamic_boost_def; // unused. exists for macro + u8 nv_dynamic_boost_min; + u8 nv_dynamic_boost_max; + u8 nv_temp_target_def; // unused. exists for macro + u8 nv_temp_target_min; + u8 nv_temp_target_max; + u8 nv_tgp_def; // unused. exists for macro + u8 nv_tgp_min; + u8 nv_tgp_max; +}; + +struct power_data { + const struct power_limits *ac_data; + const struct power_limits *dc_data; + bool requires_fan_curve; +}; + +/* + * For each avilable attribute there must be a min and a max. + * _def is not required and will be assumed to be default == max if missing. + */ +static const struct dmi_system_id power_limits[] = { + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA401W"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 75, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 30, + .ppt_pl2_sppt_min = 31, + .ppt_pl2_sppt_max = 44, + .ppt_pl3_fppt_min = 45, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507N"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 45, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80 + }, + .dc_data = NULL + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507X"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 45, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507Z"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 105, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 15, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 60, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA607P"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 30, + .ppt_pl1_spl_def = 100, + .ppt_pl1_spl_max = 135, + .ppt_pl2_sppt_min = 30, + .ppt_pl2_sppt_def = 115, + .ppt_pl2_sppt_max = 135, + .ppt_pl3_fppt_min = 30, + .ppt_pl3_fppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 115, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_def = 45, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_def = 60, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 25, + .ppt_pl3_fppt_max = 80, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA617NS"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 120 + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 25, + .ppt_apu_sppt_max = 35, + .ppt_platform_sppt_min = 45, + .ppt_platform_sppt_max = 100 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA617NT"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 115 + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 45, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 50 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA617XS"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 120, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 25, + .ppt_apu_sppt_max = 35, + .ppt_platform_sppt_min = 45, + .ppt_platform_sppt_max = 100, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FX507Z"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 15, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 60, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA401Q"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 80, + }, + .dc_data = NULL + }, + }, + { + .matches = { + // This model is full AMD. No Nvidia dGPU. + DMI_MATCH(DMI_BOARD_NAME, "GA402R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 115, + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 25, + .ppt_apu_sppt_def = 30, + .ppt_apu_sppt_max = 45, + .ppt_platform_sppt_min = 40, + .ppt_platform_sppt_max = 60, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA402X"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 35, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_def = 65, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 35, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA403U"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 65, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 35, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA503R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 35, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 65, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 25, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 60, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA605W"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 31, + .ppt_pl2_sppt_max = 44, + .ppt_pl3_fppt_min = 45, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GU603Z"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 60, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 135, + /* Only allowed in AC mode */ + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 40, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 40, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GU604V"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 65, + .ppt_pl1_spl_max = 120, + .ppt_pl2_sppt_min = 65, + .ppt_pl2_sppt_max = 150, + /* Only allowed in AC mode */ + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 40, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 40, + .ppt_pl2_sppt_max = 60, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GU605M"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 38, + .ppt_pl2_sppt_max = 53, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV301Q"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 65, + .ppt_pl2_sppt_max = 80, + }, + .dc_data = NULL + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV301R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 54, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 35, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV601R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 35, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 100, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 80, + .ppt_pl3_fppt_max = 125, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 28, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_def = 40, + .ppt_pl2_sppt_max = 60, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 80, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV601V"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_def = 100, + .ppt_pl1_spl_max = 110, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 40, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 40, + .ppt_pl2_sppt_max = 60, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GX650P"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 110, + .ppt_pl1_spl_max = 130, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 125, + .ppt_pl2_sppt_max = 130, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 125, + .ppt_pl3_fppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 25, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 35, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 42, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G513I"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + /* Yes this laptop is very limited */ + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 80, + }, + .dc_data = NULL, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G513QM"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + /* Yes this laptop is very limited */ + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 100, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 190, + }, + .dc_data = NULL, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G513R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 35, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 54, + .ppt_pl2_sppt_max = 100, + .ppt_pl3_fppt_min = 54, + .ppt_pl3_fppt_max = 125, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 50, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 50, + .ppt_pl3_fppt_min = 28, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G614J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G634J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G733C"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 170, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 35, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G733P"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 30, + .ppt_pl1_spl_def = 100, + .ppt_pl1_spl_max = 130, + .ppt_pl2_sppt_min = 65, + .ppt_pl2_sppt_def = 125, + .ppt_pl2_sppt_max = 130, + .ppt_pl3_fppt_min = 65, + .ppt_pl3_fppt_def = 125, + .ppt_pl3_fppt_max = 130, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 75, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G814J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 140, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G834J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "H7606W"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 31, + .ppt_pl2_sppt_max = 44, + .ppt_pl3_fppt_min = 45, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "RC71"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_max = 30, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 43, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_max = 53 + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_def = 15, + .ppt_pl1_spl_max = 25, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_def = 20, + .ppt_pl2_sppt_max = 30, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_def = 25, + .ppt_pl3_fppt_max = 35 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "RC72"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_max = 30, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 43, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_max = 53 + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_def = 17, + .ppt_pl1_spl_max = 25, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_def = 24, + .ppt_pl2_sppt_max = 30, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_def = 30, + .ppt_pl3_fppt_max = 35 + } + }, + }, + {} +}; + +#endif /* _ASUS_ARMOURY_H_ */ diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index f84c3d03c1de..0c6bdf93feed 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -146,8 +146,12 @@ static struct quirk_entry quirk_asus_ignore_fan = { .wmi_ignore_fan = true, }; -static struct quirk_entry quirk_asus_zenbook_duo_kbd = { - .ignore_key_wlan = true, +static struct quirk_entry quirk_asus_wlan_ignore = { + .key_wlan_event = ASUS_WMI_KEY_IGNORE, +}; + +static struct quirk_entry quirk_asus_wlan_armoury = { + .key_wlan_event = ASUS_WMI_KEY_ARMOURY, }; static int dmi_matched(const struct dmi_system_id *dmi) @@ -528,7 +532,16 @@ static const struct dmi_system_id asus_quirks[] = { DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), DMI_MATCH(DMI_PRODUCT_NAME, "UX8406MA"), }, - .driver_data = &quirk_asus_zenbook_duo_kbd, + .driver_data = &quirk_asus_wlan_ignore, + }, + { + .callback = dmi_matched, + .ident = "ASUS ROG Z13 2025", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow Z13 GZ302"), + }, + .driver_data = &quirk_asus_wlan_armoury, }, { .callback = dmi_matched, @@ -537,7 +550,7 @@ static const struct dmi_system_id asus_quirks[] = { DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), DMI_MATCH(DMI_PRODUCT_NAME, "UX8406CA"), }, - .driver_data = &quirk_asus_zenbook_duo_kbd, + .driver_data = &quirk_asus_wlan_ignore, }, {}, }; @@ -636,6 +649,7 @@ static const struct key_entry asus_nb_wmi_keymap[] = { { KE_IGNORE, 0xCF, }, /* AC mode */ { KE_KEY, 0xFA, { KEY_PROG2 } }, /* Lid flip action */ { KE_KEY, 0xBD, { KEY_PROG2 } }, /* Lid flip action on ROG xflow laptops */ + { KE_KEY, ASUS_WMI_KEY_ARMOURY, { KEY_PROG3 } }, { KE_END, 0}, }; @@ -655,11 +669,9 @@ static void asus_nb_wmi_key_filter(struct asus_wmi_driver *asus_wmi, int *code, if (atkbd_reports_vol_keys) *code = ASUS_WMI_KEY_IGNORE; break; - case 0x5D: /* Wireless console Toggle */ - case 0x5E: /* Wireless console Enable */ - case 0x5F: /* Wireless console Disable */ - if (quirks->ignore_key_wlan) - *code = ASUS_WMI_KEY_IGNORE; + case 0x5F: /* Wireless console Disable / Special Key */ + if (quirks->key_wlan_event) + *code = quirks->key_wlan_event; break; } } diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index f7191fdded14..44eeb8803669 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -55,8 +55,6 @@ module_param(fnlock_default, bool, 0444); #define to_asus_wmi_driver(pdrv) \ (container_of((pdrv), struct asus_wmi_driver, platform_driver)) -#define ASUS_WMI_MGMT_GUID "97845ED0-4E6D-11DE-8A39-0800200C9A66" - #define NOTIFY_BRNUP_MIN 0x11 #define NOTIFY_BRNUP_MAX 0x1f #define NOTIFY_BRNDOWN_MIN 0x20 @@ -105,8 +103,6 @@ module_param(fnlock_default, bool, 0444); #define USB_INTEL_XUSB2PR 0xD0 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI 0x9c31 -#define ASUS_ACPI_UID_ASUSWMI "ASUSWMI" - #define WMI_EVENT_MASK 0xFFFF #define FAN_CURVE_POINTS 8 @@ -258,6 +254,8 @@ struct asus_wmi { int tpd_led_wk; struct led_classdev kbd_led; int kbd_led_wk; + bool kbd_led_avail; + bool kbd_led_registered; struct led_classdev lightbar_led; int lightbar_led_wk; struct led_classdev micmute_led; @@ -340,6 +338,13 @@ struct asus_wmi { /* Global to allow setting externally without requiring driver data */ static enum asus_ally_mcu_hack use_ally_mcu_hack = ASUS_WMI_ALLY_MCU_HACK_INIT; +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) +static void asus_wmi_show_deprecated(void) +{ + pr_notice_once("Accessing attributes through /sys/bus/platform/asus_wmi is deprecated and will be removed in a future release. Please switch over to /sys/class/firmware_attributes.\n"); +} +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ + /* WMI ************************************************************************/ static int asus_wmi_evaluate_method3(u32 method_id, @@ -390,7 +395,7 @@ int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval) { return asus_wmi_evaluate_method3(method_id, arg0, arg1, 0, retval); } -EXPORT_SYMBOL_GPL(asus_wmi_evaluate_method); +EXPORT_SYMBOL_NS_GPL(asus_wmi_evaluate_method, "ASUS_WMI"); static int asus_wmi_evaluate_method5(u32 method_id, u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4, u32 *retval) @@ -554,12 +559,46 @@ static int asus_wmi_get_devstate(struct asus_wmi *asus, u32 dev_id, u32 *retval) return 0; } -int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, - u32 *retval) +/** + * asus_wmi_get_devstate_dsts() - Get the WMI function state. + * @dev_id: The WMI method ID to call. + * @retval: A pointer to where to store the value returned from WMI. + * @return: 0 on success and retval is filled. + * @return: -ENODEV if the method ID is unsupported. + * @return: everything else is an error from WMI call. + */ +int asus_wmi_get_devstate_dsts(u32 dev_id, u32 *retval) +{ + int err; + + err = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DSTS, dev_id, 0, retval); + if (err) + return err; + + if (*retval == ASUS_WMI_UNSUPPORTED_METHOD) + return -ENODEV; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(asus_wmi_get_devstate_dsts, "ASUS_WMI"); + +/** + * asus_wmi_set_devstate() - Set the WMI function state. + * @dev_id: The WMI function to call. + * @ctrl_param: The argument to be used for this WMI function. + * @retval: A pointer to where to store the value returned from WMI. + * @return: 0 on success and retval is filled. + * @return: everything else is an error from WMI call. + * + * A asus_wmi_set_devstate() call must be paired with a + * asus_wmi_get_devstate_dsts() to check if the WMI function is supported. + */ +int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval) { return asus_wmi_evaluate_method(ASUS_WMI_METHODID_DEVS, dev_id, ctrl_param, retval); } +EXPORT_SYMBOL_NS_GPL(asus_wmi_set_devstate, "ASUS_WMI"); /* Helper for special devices with magic return codes */ static int asus_wmi_get_devstate_bits(struct asus_wmi *asus, @@ -692,6 +731,7 @@ static void asus_wmi_tablet_mode_get_state(struct asus_wmi *asus) } /* Charging mode, 1=Barrel, 2=USB ******************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t charge_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -702,12 +742,16 @@ static ssize_t charge_mode_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", value & 0xff); } static DEVICE_ATTR_RO(charge_mode); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* dGPU ********************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t dgpu_disable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -718,6 +762,8 @@ static ssize_t dgpu_disable_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -771,8 +817,10 @@ static ssize_t dgpu_disable_store(struct device *dev, return count; } static DEVICE_ATTR_RW(dgpu_disable); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* eGPU ********************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t egpu_enable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -783,6 +831,8 @@ static ssize_t egpu_enable_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -839,8 +889,10 @@ static ssize_t egpu_enable_store(struct device *dev, return count; } static DEVICE_ATTR_RW(egpu_enable); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Is eGPU connected? *********************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t egpu_connected_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -851,12 +903,16 @@ static ssize_t egpu_connected_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } static DEVICE_ATTR_RO(egpu_connected); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* gpu mux switch *************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t gpu_mux_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -867,6 +923,8 @@ static ssize_t gpu_mux_mode_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -925,6 +983,7 @@ static ssize_t gpu_mux_mode_store(struct device *dev, return count; } static DEVICE_ATTR_RW(gpu_mux_mode); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* TUF Laptop Keyboard RGB Modes **********************************************/ static ssize_t kbd_rgb_mode_store(struct device *dev, @@ -1048,6 +1107,7 @@ static const struct attribute_group *kbd_rgb_mode_groups[] = { }; /* Tunable: PPT: Intel=PL1, AMD=SPPT *****************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t ppt_pl2_sppt_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -1086,6 +1146,8 @@ static ssize_t ppt_pl2_sppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_pl2_sppt); } static DEVICE_ATTR_RW(ppt_pl2_sppt); @@ -1128,6 +1190,8 @@ static ssize_t ppt_pl1_spl_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_pl1_spl); } static DEVICE_ATTR_RW(ppt_pl1_spl); @@ -1171,6 +1235,8 @@ static ssize_t ppt_fppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_fppt); } static DEVICE_ATTR_RW(ppt_fppt); @@ -1214,6 +1280,8 @@ static ssize_t ppt_apu_sppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_apu_sppt); } static DEVICE_ATTR_RW(ppt_apu_sppt); @@ -1257,6 +1325,8 @@ static ssize_t ppt_platform_sppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_platform_sppt); } static DEVICE_ATTR_RW(ppt_platform_sppt); @@ -1300,6 +1370,8 @@ static ssize_t nv_dynamic_boost_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->nv_dynamic_boost); } static DEVICE_ATTR_RW(nv_dynamic_boost); @@ -1343,9 +1415,12 @@ static ssize_t nv_temp_target_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->nv_temp_target); } static DEVICE_ATTR_RW(nv_temp_target); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Ally MCU Powersave ********************************************************/ @@ -1386,6 +1461,7 @@ void set_ally_mcu_powersave(bool enabled) } EXPORT_SYMBOL_NS_GPL(set_ally_mcu_powersave, "ASUS_WMI"); +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t mcu_powersave_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1396,6 +1472,8 @@ static ssize_t mcu_powersave_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -1431,6 +1509,7 @@ static ssize_t mcu_powersave_store(struct device *dev, return count; } static DEVICE_ATTR_RW(mcu_powersave); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Battery ********************************************************************/ @@ -1530,6 +1609,92 @@ static void asus_wmi_battery_exit(struct asus_wmi *asus) /* LEDs ***********************************************************************/ +struct asus_hid_ref { + struct list_head listeners; + struct asus_wmi *asus; + spinlock_t lock; +}; + +struct asus_hid_ref asus_ref = { + .listeners = LIST_HEAD_INIT(asus_ref.listeners), + .asus = NULL, + .lock = __SPIN_LOCK_UNLOCKED(asus_ref.lock), +}; + +int asus_hid_register_listener(struct asus_hid_listener *bdev) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&asus_ref.lock, flags); + list_add_tail(&bdev->list, &asus_ref.listeners); + if (asus_ref.asus) { + if (asus_ref.asus->kbd_led_registered && asus_ref.asus->kbd_led_wk >= 0) + bdev->brightness_set(bdev, asus_ref.asus->kbd_led_wk); + + if (!asus_ref.asus->kbd_led_registered) { + ret = led_classdev_register( + &asus_ref.asus->platform_device->dev, + &asus_ref.asus->kbd_led); + if (!ret) + asus_ref.asus->kbd_led_registered = true; + } + } + spin_unlock_irqrestore(&asus_ref.lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(asus_hid_register_listener); + +void asus_hid_unregister_listener(struct asus_hid_listener *bdev) +{ + unsigned long flags; + + spin_lock_irqsave(&asus_ref.lock, flags); + list_del(&bdev->list); + spin_unlock_irqrestore(&asus_ref.lock, flags); +} +EXPORT_SYMBOL_GPL(asus_hid_unregister_listener); + +static void do_kbd_led_set(struct led_classdev *led_cdev, int value); + +int asus_hid_event(enum asus_hid_event event) +{ + unsigned long flags; + int brightness; + + spin_lock_irqsave(&asus_ref.lock, flags); + if (!asus_ref.asus || !asus_ref.asus->kbd_led_registered) { + spin_unlock_irqrestore(&asus_ref.lock, flags); + return -EBUSY; + } + brightness = asus_ref.asus->kbd_led_wk; + + switch (event) { + case ASUS_EV_BRTUP: + brightness += 1; + break; + case ASUS_EV_BRTDOWN: + brightness -= 1; + break; + case ASUS_EV_BRTTOGGLE: + if (brightness >= ASUS_EV_MAX_BRIGHTNESS) + brightness = 0; + else + brightness += 1; + break; + } + + do_kbd_led_set(&asus_ref.asus->kbd_led, brightness); + led_classdev_notify_brightness_hw_changed(&asus_ref.asus->kbd_led, + asus_ref.asus->kbd_led_wk); + + spin_unlock_irqrestore(&asus_ref.lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(asus_hid_event); + /* * These functions actually update the LED's, and are called from a * workqueue. By doing this as separate work rather than when the LED @@ -1609,6 +1774,7 @@ static int kbd_led_read(struct asus_wmi *asus, int *level, int *env) static void do_kbd_led_set(struct led_classdev *led_cdev, int value) { + struct asus_hid_listener *listener; struct asus_wmi *asus; int max_level; @@ -1616,25 +1782,39 @@ static void do_kbd_led_set(struct led_classdev *led_cdev, int value) max_level = asus->kbd_led.max_brightness; asus->kbd_led_wk = clamp_val(value, 0, max_level); - kbd_led_update(asus); + + if (asus->kbd_led_avail) + kbd_led_update(asus); + + list_for_each_entry(listener, &asus_ref.listeners, list) + listener->brightness_set(listener, asus->kbd_led_wk); } static void kbd_led_set(struct led_classdev *led_cdev, enum led_brightness value) { + unsigned long flags; + /* Prevent disabling keyboard backlight on module unregister */ if (led_cdev->flags & LED_UNREGISTERING) return; + spin_lock_irqsave(&asus_ref.lock, flags); do_kbd_led_set(led_cdev, value); + spin_unlock_irqrestore(&asus_ref.lock, flags); } static void kbd_led_set_by_kbd(struct asus_wmi *asus, enum led_brightness value) { - struct led_classdev *led_cdev = &asus->kbd_led; + struct led_classdev *led_cdev; + unsigned long flags; + + spin_lock_irqsave(&asus_ref.lock, flags); + led_cdev = &asus->kbd_led; do_kbd_led_set(led_cdev, value); led_classdev_notify_brightness_hw_changed(led_cdev, asus->kbd_led_wk); + spin_unlock_irqrestore(&asus_ref.lock, flags); } static enum led_brightness kbd_led_get(struct led_classdev *led_cdev) @@ -1644,6 +1824,9 @@ static enum led_brightness kbd_led_get(struct led_classdev *led_cdev) asus = container_of(led_cdev, struct asus_wmi, kbd_led); + if (!asus->kbd_led_avail) + return asus->kbd_led_wk; + retval = kbd_led_read(asus, &value, NULL); if (retval < 0) return retval; @@ -1759,7 +1942,15 @@ static int camera_led_set(struct led_classdev *led_cdev, static void asus_wmi_led_exit(struct asus_wmi *asus) { - led_classdev_unregister(&asus->kbd_led); + unsigned long flags; + + spin_lock_irqsave(&asus_ref.lock, flags); + asus_ref.asus = NULL; + spin_unlock_irqrestore(&asus_ref.lock, flags); + + if (asus->kbd_led_registered) + led_classdev_unregister(&asus->kbd_led); + led_classdev_unregister(&asus->tpd_led); led_classdev_unregister(&asus->wlan_led); led_classdev_unregister(&asus->lightbar_led); @@ -1773,6 +1964,8 @@ static void asus_wmi_led_exit(struct asus_wmi *asus) static int asus_wmi_led_init(struct asus_wmi *asus) { int rv = 0, num_rgb_groups = 0, led_val; + struct asus_hid_listener *listener; + unsigned long flags; if (asus->kbd_rgb_dev) kbd_rgb_mode_groups[num_rgb_groups++] = &kbd_rgb_mode_group; @@ -1797,23 +1990,38 @@ static int asus_wmi_led_init(struct asus_wmi *asus) goto error; } - if (!kbd_led_read(asus, &led_val, NULL) && !dmi_check_system(asus_use_hid_led_dmi_ids)) { - pr_info("using asus-wmi for asus::kbd_backlight\n"); + asus->kbd_led.name = "asus::kbd_backlight"; + asus->kbd_led.flags = LED_BRIGHT_HW_CHANGED; + asus->kbd_led.brightness_set = kbd_led_set; + asus->kbd_led.brightness_get = kbd_led_get; + asus->kbd_led.max_brightness = ASUS_EV_MAX_BRIGHTNESS; + asus->kbd_led_avail = !kbd_led_read(asus, &led_val, NULL); + + if (asus->kbd_led_avail) asus->kbd_led_wk = led_val; - asus->kbd_led.name = "asus::kbd_backlight"; - asus->kbd_led.flags = LED_BRIGHT_HW_CHANGED; - asus->kbd_led.brightness_set = kbd_led_set; - asus->kbd_led.brightness_get = kbd_led_get; - asus->kbd_led.max_brightness = 3; + else + asus->kbd_led_wk = -1; - if (num_rgb_groups != 0) - asus->kbd_led.groups = kbd_rgb_mode_groups; + if (asus->kbd_led_avail && num_rgb_groups != 0) + asus->kbd_led.groups = kbd_rgb_mode_groups; + spin_lock_irqsave(&asus_ref.lock, flags); + if (asus->kbd_led_avail || !list_empty(&asus_ref.listeners)) { rv = led_classdev_register(&asus->platform_device->dev, &asus->kbd_led); - if (rv) + if (rv) { + spin_unlock_irqrestore(&asus_ref.lock, flags); goto error; + } + asus->kbd_led_registered = true; + + if (asus->kbd_led_wk >= 0) { + list_for_each_entry(listener, &asus_ref.listeners, list) + listener->brightness_set(listener, asus->kbd_led_wk); + } } + asus_ref.asus = asus; + spin_unlock_irqrestore(&asus_ref.lock, flags); if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_WIRELESS_LED) && (asus->driver->quirks->wapf > 0)) { @@ -2304,6 +2512,7 @@ static int asus_wmi_rfkill_init(struct asus_wmi *asus) } /* Panel Overdrive ************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t panel_od_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2314,6 +2523,8 @@ static ssize_t panel_od_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -2350,9 +2561,10 @@ static ssize_t panel_od_store(struct device *dev, return count; } static DEVICE_ATTR_RW(panel_od); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Bootup sound ***************************************************************/ - +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t boot_sound_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2363,6 +2575,8 @@ static ssize_t boot_sound_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -2398,8 +2612,10 @@ static ssize_t boot_sound_store(struct device *dev, return count; } static DEVICE_ATTR_RW(boot_sound); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Mini-LED mode **************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t mini_led_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2430,6 +2646,8 @@ static ssize_t mini_led_mode_show(struct device *dev, } } + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", value); } @@ -2500,10 +2718,13 @@ static ssize_t available_mini_led_mode_show(struct device *dev, return sysfs_emit(buf, "0 1 2\n"); } + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "0\n"); } static DEVICE_ATTR_RO(available_mini_led_mode); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Quirks *********************************************************************/ @@ -3791,6 +4012,7 @@ static int throttle_thermal_policy_set_default(struct asus_wmi *asus) return throttle_thermal_policy_write(asus); } +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t throttle_thermal_policy_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -3834,6 +4056,7 @@ static ssize_t throttle_thermal_policy_store(struct device *dev, * Throttle thermal policy: 0 - default, 1 - overboost, 2 - silent */ static DEVICE_ATTR_RW(throttle_thermal_policy); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Platform profile ***********************************************************/ static int asus_wmi_platform_profile_get(struct device *dev, @@ -3853,7 +4076,7 @@ static int asus_wmi_platform_profile_get(struct device *dev, *profile = PLATFORM_PROFILE_PERFORMANCE; break; case ASUS_THROTTLE_THERMAL_POLICY_SILENT: - *profile = PLATFORM_PROFILE_QUIET; + *profile = PLATFORM_PROFILE_LOW_POWER; break; default: return -EINVAL; @@ -3877,7 +4100,7 @@ static int asus_wmi_platform_profile_set(struct device *dev, case PLATFORM_PROFILE_BALANCED: tp = ASUS_THROTTLE_THERMAL_POLICY_DEFAULT; break; - case PLATFORM_PROFILE_QUIET: + case PLATFORM_PROFILE_LOW_POWER: tp = ASUS_THROTTLE_THERMAL_POLICY_SILENT; break; default: @@ -3890,7 +4113,7 @@ static int asus_wmi_platform_profile_set(struct device *dev, static int asus_wmi_platform_profile_probe(void *drvdata, unsigned long *choices) { - set_bit(PLATFORM_PROFILE_QUIET, choices); + set_bit(PLATFORM_PROFILE_LOW_POWER, choices); set_bit(PLATFORM_PROFILE_BALANCED, choices); set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); @@ -4435,27 +4658,29 @@ static struct attribute *platform_attributes[] = { &dev_attr_camera.attr, &dev_attr_cardr.attr, &dev_attr_touchpad.attr, - &dev_attr_charge_mode.attr, - &dev_attr_egpu_enable.attr, - &dev_attr_egpu_connected.attr, - &dev_attr_dgpu_disable.attr, - &dev_attr_gpu_mux_mode.attr, &dev_attr_lid_resume.attr, &dev_attr_als_enable.attr, &dev_attr_fan_boost_mode.attr, - &dev_attr_throttle_thermal_policy.attr, - &dev_attr_ppt_pl2_sppt.attr, - &dev_attr_ppt_pl1_spl.attr, - &dev_attr_ppt_fppt.attr, - &dev_attr_ppt_apu_sppt.attr, - &dev_attr_ppt_platform_sppt.attr, - &dev_attr_nv_dynamic_boost.attr, - &dev_attr_nv_temp_target.attr, - &dev_attr_mcu_powersave.attr, - &dev_attr_boot_sound.attr, - &dev_attr_panel_od.attr, - &dev_attr_mini_led_mode.attr, - &dev_attr_available_mini_led_mode.attr, +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) + &dev_attr_charge_mode.attr, + &dev_attr_egpu_enable.attr, + &dev_attr_egpu_connected.attr, + &dev_attr_dgpu_disable.attr, + &dev_attr_gpu_mux_mode.attr, + &dev_attr_ppt_pl2_sppt.attr, + &dev_attr_ppt_pl1_spl.attr, + &dev_attr_ppt_fppt.attr, + &dev_attr_ppt_apu_sppt.attr, + &dev_attr_ppt_platform_sppt.attr, + &dev_attr_nv_dynamic_boost.attr, + &dev_attr_nv_temp_target.attr, + &dev_attr_mcu_powersave.attr, + &dev_attr_boot_sound.attr, + &dev_attr_panel_od.attr, + &dev_attr_mini_led_mode.attr, + &dev_attr_available_mini_led_mode.attr, + &dev_attr_throttle_thermal_policy.attr, +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ NULL }; @@ -4477,7 +4702,11 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj, devid = ASUS_WMI_DEVID_LID_RESUME; else if (attr == &dev_attr_als_enable.attr) devid = ASUS_WMI_DEVID_ALS_ENABLE; - else if (attr == &dev_attr_charge_mode.attr) + else if (attr == &dev_attr_fan_boost_mode.attr) + ok = asus->fan_boost_mode_available; + +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) + if (attr == &dev_attr_charge_mode.attr) devid = ASUS_WMI_DEVID_CHARGE_MODE; else if (attr == &dev_attr_egpu_enable.attr) ok = asus->egpu_enable_available; @@ -4515,6 +4744,7 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj, ok = asus->mini_led_dev_id != 0; else if (attr == &dev_attr_available_mini_led_mode.attr) ok = asus->mini_led_dev_id != 0; +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ if (devid != -1) { ok = !(asus_wmi_get_devstate_simple(asus, devid) < 0); @@ -4770,6 +5000,7 @@ static int asus_wmi_add(struct platform_device *pdev) } /* ensure defaults for tunables */ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) asus->ppt_pl2_sppt = 5; asus->ppt_pl1_spl = 5; asus->ppt_apu_sppt = 5; @@ -4792,17 +5023,18 @@ static int asus_wmi_add(struct platform_device *pdev) asus->gpu_mux_dev = ASUS_WMI_DEVID_GPU_MUX; else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_GPU_MUX_VIVO)) asus->gpu_mux_dev = ASUS_WMI_DEVID_GPU_MUX_VIVO; - - if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE)) - asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE; - else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE2)) - asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE2; +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY)) asus->throttle_thermal_policy_dev = ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY; else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY_VIVO)) asus->throttle_thermal_policy_dev = ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY_VIVO; + if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE)) + asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE; + else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE2)) + asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE2; + err = fan_boost_mode_check_present(asus); if (err) goto fail_fan_boost_mode; diff --git a/drivers/platform/x86/asus-wmi.h b/drivers/platform/x86/asus-wmi.h index 018dfde4025e..5cd4392b964e 100644 --- a/drivers/platform/x86/asus-wmi.h +++ b/drivers/platform/x86/asus-wmi.h @@ -18,6 +18,7 @@ #include #define ASUS_WMI_KEY_IGNORE (-1) +#define ASUS_WMI_KEY_ARMOURY 0xffff01 #define ASUS_WMI_BRN_DOWN 0x2e #define ASUS_WMI_BRN_UP 0x2f @@ -40,7 +41,7 @@ struct quirk_entry { bool wmi_force_als_set; bool wmi_ignore_fan; bool filter_i8042_e1_extended_codes; - bool ignore_key_wlan; + int key_wlan_event; enum asus_wmi_tablet_switch_mode tablet_switch_mode; int wapf; /* diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 8a515179113d..bf428546f0a4 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -6,6 +6,9 @@ #include #include +#define ASUS_WMI_MGMT_GUID "97845ED0-4E6D-11DE-8A39-0800200C9A66" +#define ASUS_ACPI_UID_ASUSWMI "ASUSWMI" + /* WMI Methods */ #define ASUS_WMI_METHODID_SPEC 0x43455053 /* BIOS SPECification */ #define ASUS_WMI_METHODID_SFBD 0x44424653 /* Set First Boot Device */ @@ -73,12 +76,14 @@ #define ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY_VIVO 0x00110019 /* Misc */ +#define ASUS_WMI_DEVID_PANEL_HD 0x0005001C #define ASUS_WMI_DEVID_PANEL_OD 0x00050019 #define ASUS_WMI_DEVID_CAMERA 0x00060013 #define ASUS_WMI_DEVID_LID_FLIP 0x00060062 #define ASUS_WMI_DEVID_LID_FLIP_ROG 0x00060077 #define ASUS_WMI_DEVID_MINI_LED_MODE 0x0005001E #define ASUS_WMI_DEVID_MINI_LED_MODE2 0x0005002E +#define ASUS_WMI_DEVID_SCREEN_AUTO_BRIGHTNESS 0x0005002A /* Storage */ #define ASUS_WMI_DEVID_CARDREADER 0x00080013 @@ -133,6 +138,16 @@ /* dgpu on/off */ #define ASUS_WMI_DEVID_DGPU 0x00090020 +/* Intel E-core and P-core configuration in a format 0x0[E]0[P] */ +#define ASUS_WMI_DEVID_CORES 0x001200D2 + /* Maximum Intel E-core and P-core availability */ +#define ASUS_WMI_DEVID_CORES_MAX 0x001200D3 + +#define ASUS_WMI_DEVID_APU_MEM 0x000600C1 + +#define ASUS_WMI_DEVID_DGPU_BASE_TGP 0x00120099 +#define ASUS_WMI_DEVID_DGPU_SET_TGP 0x00120098 + /* gpu mux switch, 0 = dGPU, 1 = Optimus */ #define ASUS_WMI_DEVID_GPU_MUX 0x00090016 #define ASUS_WMI_DEVID_GPU_MUX_VIVO 0x00090026 @@ -163,11 +178,29 @@ enum asus_ally_mcu_hack { ASUS_WMI_ALLY_MCU_HACK_DISABLED, }; +struct asus_hid_listener { + struct list_head list; + void (*brightness_set)(struct asus_hid_listener *listener, int brightness); +}; + +enum asus_hid_event { + ASUS_EV_BRTUP, + ASUS_EV_BRTDOWN, + ASUS_EV_BRTTOGGLE, +}; + +#define ASUS_EV_MAX_BRIGHTNESS 3 + #if IS_REACHABLE(CONFIG_ASUS_WMI) void set_ally_mcu_hack(enum asus_ally_mcu_hack status); void set_ally_mcu_powersave(bool enabled); +int asus_wmi_get_devstate_dsts(u32 dev_id, u32 *retval); int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval); int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval); + +int asus_hid_register_listener(struct asus_hid_listener *cdev); +void asus_hid_unregister_listener(struct asus_hid_listener *cdev); +int asus_hid_event(enum asus_hid_event event); #else static inline void set_ally_mcu_hack(enum asus_ally_mcu_hack status) { @@ -179,14 +212,31 @@ static inline int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval) { return -ENODEV; } +static inline int asus_wmi_get_devstate_dsts(u32 dev_id, u32 *retval) +{ + return -ENODEV; +} static inline int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval) { return -ENODEV; } + +static inline int asus_hid_register_listener(struct asus_hid_listener *bdev) +{ + return -ENODEV; +} +static inline void asus_hid_unregister_listener(struct asus_hid_listener *bdev) +{ +} +static inline int asus_hid_event(enum asus_hid_event event) +{ + return -ENODEV; +} #endif /* To be used by both hid-asus and asus-wmi to determine which controls kbd_brightness */ +#if IS_REACHABLE(CONFIG_ASUS_WMI) || IS_REACHABLE(CONFIG_HID_ASUS) static const struct dmi_system_id asus_use_hid_led_dmi_ids[] = { { .matches = { @@ -225,5 +275,6 @@ static const struct dmi_system_id asus_use_hid_led_dmi_ids[] = { }, { }, }; +#endif #endif /* __PLATFORM_DATA_X86_ASUS_WMI_H */ -- 2.51.0 From f134ee18c49208ae3b5f56086ab0e2862c5ad8ff Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 18:19:23 +0200 Subject: [PATCH 2/6] bbr3 Signed-off-by: Peter Jung --- include/linux/tcp.h | 6 +- include/net/inet_connection_sock.h | 4 +- include/net/tcp.h | 73 +- include/uapi/linux/inet_diag.h | 23 + include/uapi/linux/rtnetlink.h | 4 +- include/uapi/linux/tcp.h | 1 + net/ipv4/Kconfig | 21 +- net/ipv4/bpf_tcp_ca.c | 4 +- net/ipv4/tcp.c | 3 + net/ipv4/tcp_bbr.c | 2232 +++++++++++++++++++++------- net/ipv4/tcp_cong.c | 1 + net/ipv4/tcp_input.c | 40 +- net/ipv4/tcp_minisocks.c | 2 + net/ipv4/tcp_output.c | 48 +- net/ipv4/tcp_rate.c | 30 +- net/ipv4/tcp_timer.c | 4 +- 16 files changed, 1941 insertions(+), 555 deletions(-) diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 29f59d50dc73..811850c240cc 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -248,7 +248,8 @@ struct tcp_sock { void (*tcp_clean_acked)(struct sock *sk, u32 acked_seq); #endif u32 snd_ssthresh; /* Slow start size threshold */ - u8 recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ + u32 recvmsg_inq : 1,/* Indicate # of bytes in queue upon recvmsg */ + fast_ack_mode:1;/* ack ASAP if >1 rcv_mss received? */ __cacheline_group_end(tcp_sock_read_rx); /* TX read-write hotpath cache lines */ @@ -305,7 +306,8 @@ struct tcp_sock { */ struct tcp_options_received rx_opt; u8 nonagle : 4,/* Disable Nagle algorithm? */ - rate_app_limited:1; /* rate_{delivered,interval_us} limited? */ + rate_app_limited:1, /* rate_{delivered,interval_us} limited? */ + tlp_orig_data_app_limited:1; /* app-limited before TLP rtx? */ __cacheline_group_end(tcp_sock_write_txrx); /* RX read-write hotpath cache lines */ diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 1735db332aab..2c4a94af7093 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -132,8 +132,8 @@ struct inet_connection_sock { u32 icsk_probes_tstamp; u32 icsk_user_timeout; - u64 icsk_ca_priv[104 / sizeof(u64)]; -#define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) +#define ICSK_CA_PRIV_SIZE (144) + u64 icsk_ca_priv[ICSK_CA_PRIV_SIZE / sizeof(u64)]; }; #define ICSK_TIME_RETRANS 1 /* Retransmit timer */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 5078ad868fee..de404e4370d4 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -379,11 +379,14 @@ static inline void tcp_dec_quickack_mode(struct sock *sk) #define TCP_ECN_DEMAND_CWR BIT(2) #define TCP_ECN_SEEN BIT(3) #define TCP_ECN_MODE_ACCECN BIT(4) +#define TCP_ECN_LOW BIT(5) +#define TCP_ECN_ECT_PERMANENT BIT(6) #define TCP_ECN_DISABLED 0 #define TCP_ECN_MODE_PENDING (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) #define TCP_ECN_MODE_ANY (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) + static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp) { return tp->ecn_flags & TCP_ECN_MODE_ANY; @@ -841,6 +844,15 @@ static inline void tcp_fast_path_check(struct sock *sk) u32 tcp_delack_max(const struct sock *sk); +static inline void tcp_set_ecn_low_from_dst(struct sock *sk, + const struct dst_entry *dst) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (dst_feature(dst, RTAX_FEATURE_ECN_LOW)) + tp->ecn_flags |= TCP_ECN_LOW; +} + /* Compute the actual rto_min value */ static inline u32 tcp_rto_min(const struct sock *sk) { @@ -946,6 +958,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) return max_t(s64, t1 - t0, 0); } +static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) +{ + return max_t(s32, t1 - t0, 0); +} + /* provide the departure time in us unit */ static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) { @@ -1044,9 +1061,14 @@ struct tcp_skb_cb { /* pkts S/ACKed so far upon tx of skb, incl retrans: */ __u32 delivered; /* start of send pipeline phase */ - u64 first_tx_mstamp; + u32 first_tx_mstamp; /* when we reached the "delivered" count */ - u64 delivered_mstamp; + u32 delivered_mstamp; +#define TCPCB_IN_FLIGHT_BITS 20 +#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) + u32 in_flight:20, /* packets in flight at transmit */ + unused2:12; + u32 lost; /* packets lost so far upon tx of skb */ } tx; /* only used for outgoing skbs */ union { struct inet_skb_parm h4; @@ -1159,6 +1181,7 @@ enum tcp_ca_event { CA_EVENT_LOSS, /* loss timeout */ CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ + CA_EVENT_TLP_RECOVERY, /* a lost segment was repaired by TLP probe */ }; /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ @@ -1181,7 +1204,11 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ +#define TCP_CONG_WANTS_CE_EVENTS BIT(2) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ + TCP_CONG_NEEDS_ECN | \ + TCP_CONG_WANTS_CE_EVENTS) union tcp_cc_info; @@ -1201,10 +1228,13 @@ struct ack_sample { */ struct rate_sample { u64 prior_mstamp; /* starting timestamp for interval */ + u32 prior_lost; /* tp->lost at "prior_mstamp" */ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ + u32 tx_in_flight; /* packets in flight at starting timestamp */ + s32 lost; /* number of packets lost over interval */ s32 delivered; /* number of packets delivered over interval */ - s32 delivered_ce; /* number of packets delivered w/ CE marks*/ + s32 delivered_ce; /* packets delivered w/ CE mark over interval */ long interval_us; /* time for tp->delivered to incr "delivered" */ u32 snd_interval_us; /* snd interval for delivered packets */ u32 rcv_interval_us; /* rcv interval for delivered packets */ @@ -1215,7 +1245,9 @@ struct rate_sample { u32 last_end_seq; /* end_seq of most recently ACKed packet */ bool is_app_limited; /* is sample from packet with bubble in pipe? */ bool is_retrans; /* is sample from retransmission? */ + bool is_acking_tlp_retrans_seq; /* ACKed a TLP retransmit sequence? */ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + bool is_ece; /* did this ACK have ECN marked? */ }; struct tcp_congestion_ops { @@ -1239,8 +1271,11 @@ struct tcp_congestion_ops { /* hook for packet ack accounting (optional) */ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - /* override sysctl_tcp_min_tso_segs */ - u32 (*min_tso_segs)(struct sock *sk); + /* pick target number of segments per TSO/GSO skb (optional): */ + u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); + + /* react to a specific lost skb (optional) */ + void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); /* call when packets are delivered to update cwnd and pacing rate, * after all the ca_state processing. (optional) @@ -1306,6 +1341,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) } #endif +static inline bool tcp_ca_wants_ce_events(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | + TCP_CONG_WANTS_CE_EVENTS); +} + static inline bool tcp_ca_needs_ecn(const struct sock *sk) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -1325,6 +1368,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) void tcp_set_ca_state(struct sock *sk, const u8 ca_state); /* From tcp_rate.c */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, struct rate_sample *rs); @@ -1337,6 +1381,21 @@ static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) return t1 > t2 || (t1 == t2 && after(seq1, seq2)); } +/* If a retransmit failed due to local qdisc congestion or other local issues, + * then we may have called tcp_set_skb_tso_segs() to increase the number of + * segments in the skb without increasing the tx.in_flight. In all other cases, + * the tx.in_flight should be at least as big as the pcount of the sk_buff. We + * do not have the state to know whether a retransmit failed due to local qdisc + * congestion or other local issues, so to avoid spurious warnings we consider + * that any skb marked lost may have suffered that fate. + */ +static inline bool tcp_skb_tx_in_flight_is_suspicious(u32 skb_pcount, + u32 skb_sacked_flags, + u32 tx_in_flight) +{ + return (skb_pcount > tx_in_flight) && !(skb_sacked_flags & TCPCB_LOST); +} + /* These functions determine how the current flow behaves in respect of SACK * handling. SACK is negotiated with the peer, and therefore it can vary * between different flows. @@ -2490,7 +2549,7 @@ struct tcp_plb_state { u8 consec_cong_rounds:5, /* consecutive congested rounds */ unused:3; u32 pause_until; /* jiffies32 when PLB can resume rerouting */ -}; +} __attribute__ ((__packed__)); static inline void tcp_plb_init(const struct sock *sk, struct tcp_plb_state *plb) diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h index 86bb2e8b17c9..9d9a3eb2ce9b 100644 --- a/include/uapi/linux/inet_diag.h +++ b/include/uapi/linux/inet_diag.h @@ -229,6 +229,29 @@ struct tcp_bbr_info { __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ + __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ + __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ + __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ + __u8 bbr_mode; /* current bbr_mode in state machine */ + __u8 bbr_phase; /* current state machine phase */ + __u8 unused1; /* alignment padding; not used yet */ + __u8 bbr_version; /* BBR algorithm version */ + __u32 bbr_inflight_lo; /* lower short-term data volume bound */ + __u32 bbr_inflight_hi; /* higher long-term data volume bound */ + __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ +}; + +/* TCP BBR congestion control bbr_phase as reported in netlink/ss stats. */ +enum tcp_bbr_phase { + BBR_PHASE_INVALID = 0, + BBR_PHASE_STARTUP = 1, + BBR_PHASE_DRAIN = 2, + BBR_PHASE_PROBE_RTT = 3, + BBR_PHASE_PROBE_BW_UP = 4, + BBR_PHASE_PROBE_BW_DOWN = 5, + BBR_PHASE_PROBE_BW_CRUISE = 6, + BBR_PHASE_PROBE_BW_REFILL = 7, }; union tcp_cc_info { diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h index dab9493c791b..cce4975fdcfe 100644 --- a/include/uapi/linux/rtnetlink.h +++ b/include/uapi/linux/rtnetlink.h @@ -517,12 +517,14 @@ enum { #define RTAX_FEATURE_TIMESTAMP (1 << 2) /* unused */ #define RTAX_FEATURE_ALLFRAG (1 << 3) /* unused */ #define RTAX_FEATURE_TCP_USEC_TS (1 << 4) +#define RTAX_FEATURE_ECN_LOW (1 << 5) #define RTAX_FEATURE_MASK (RTAX_FEATURE_ECN | \ RTAX_FEATURE_SACK | \ RTAX_FEATURE_TIMESTAMP | \ RTAX_FEATURE_ALLFRAG | \ - RTAX_FEATURE_TCP_USEC_TS) + RTAX_FEATURE_TCP_USEC_TS | \ + RTAX_FEATURE_ECN_LOW) struct rta_session { __u8 proto; diff --git a/include/uapi/linux/tcp.h b/include/uapi/linux/tcp.h index bdac8c42fa82..362644a272ba 100644 --- a/include/uapi/linux/tcp.h +++ b/include/uapi/linux/tcp.h @@ -185,6 +185,7 @@ enum tcp_fastopen_client_fail { #define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */ #define TCPI_OPT_USEC_TS 64 /* usec timestamps */ #define TCPI_OPT_TFO_CHILD 128 /* child from a Fast Open option on SYN */ +#define TCPI_OPT_ECN_LOW 256 /* Low-latency ECN configured at init */ /* * Sender's congestion state indicating normal or abnormal situations diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index 12850a277251..3b8b96692fb4 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -669,15 +669,18 @@ config TCP_CONG_BBR default n help - BBR (Bottleneck Bandwidth and RTT) TCP congestion control aims to - maximize network utilization and minimize queues. It builds an explicit - model of the bottleneck delivery rate and path round-trip propagation - delay. It tolerates packet loss and delay unrelated to congestion. It - can operate over LAN, WAN, cellular, wifi, or cable modem links. It can - coexist with flows that use loss-based congestion control, and can - operate with shallow buffers, deep buffers, bufferbloat, policers, or - AQM schemes that do not provide a delay signal. It requires the fq - ("Fair Queue") pacing packet scheduler. + BBR (Bottleneck Bandwidth and RTT) TCP congestion control is a + model-based congestion control algorithm that aims to maximize + network utilization, keep queues and retransmit rates low, and to be + able to coexist with Reno/CUBIC in common scenarios. It builds an + explicit model of the network path. It tolerates a targeted degree + of random packet loss and delay. It can operate over LAN, WAN, + cellular, wifi, or cable modem links, and can use shallow-threshold + ECN signals. It can coexist to some degree with flows that use + loss-based congestion control, and can operate with shallow buffers, + deep buffers, bufferbloat, policers, or AQM schemes that do not + provide a delay signal. It requires pacing, using either TCP internal + pacing or the fq ("Fair Queue") pacing packet scheduler. choice prompt "Default TCP congestion control" diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index e01492234b0b..27893b774e08 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -280,7 +280,7 @@ static void bpf_tcp_ca_pkts_acked(struct sock *sk, const struct ack_sample *samp { } -static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk) +static u32 bpf_tcp_ca_tso_segs(struct sock *sk, unsigned int mss_now) { return 0; } @@ -315,7 +315,7 @@ static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = { .cwnd_event = bpf_tcp_ca_cwnd_event, .in_ack_event = bpf_tcp_ca_in_ack_event, .pkts_acked = bpf_tcp_ca_pkts_acked, - .min_tso_segs = bpf_tcp_ca_min_tso_segs, + .tso_segs = bpf_tcp_ca_tso_segs, .cong_control = bpf_tcp_ca_cong_control, .undo_cwnd = bpf_tcp_ca_undo_cwnd, .sndbuf_expand = bpf_tcp_ca_sndbuf_expand, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 461a9ab540af..02ae796fa17e 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3442,6 +3442,7 @@ int tcp_disconnect(struct sock *sk, int flags) tp->rx_opt.dsack = 0; tp->rx_opt.num_sacks = 0; tp->rcv_ooopack = 0; + tp->fast_ack_mode = 0; /* Clean up fastopen related fields */ @@ -4189,6 +4190,8 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info) info->tcpi_options |= TCPI_OPT_ECN; if (tp->ecn_flags & TCP_ECN_SEEN) info->tcpi_options |= TCPI_OPT_ECN_SEEN; + if (tp->ecn_flags & TCP_ECN_LOW) + info->tcpi_options |= TCPI_OPT_ECN_LOW; if (tp->syn_data_acked) info->tcpi_options |= TCPI_OPT_SYN_DATA; if (tp->tcp_usec_ts) diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 760941e55153..066da5e5747c 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -1,18 +1,19 @@ -/* Bottleneck Bandwidth and RTT (BBR) congestion control +/* BBR (Bottleneck Bandwidth and RTT) congestion control * - * BBR congestion control computes the sending rate based on the delivery - * rate (throughput) estimated from ACKs. In a nutshell: + * BBR is a model-based congestion control algorithm that aims for low queues, + * low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model of the + * network path, it uses measurements of bandwidth and RTT, as well as (if they + * occur) packet loss and/or shallow-threshold ECN signals. Note that although + * it can use ECN or loss signals explicitly, it does not require either; it + * can bound its in-flight data based on its estimate of the BDP. * - * On each ACK, update our model of the network path: - * bottleneck_bandwidth = windowed_max(delivered / elapsed, 10 round trips) - * min_rtt = windowed_min(rtt, 10 seconds) - * pacing_rate = pacing_gain * bottleneck_bandwidth - * cwnd = max(cwnd_gain * bottleneck_bandwidth * min_rtt, 4) - * - * The core algorithm does not react directly to packet losses or delays, - * although BBR may adjust the size of next send per ACK when loss is - * observed, or adjust the sending rate if it estimates there is a - * traffic policer, in order to keep the drop rate reasonable. + * The model has both higher and lower bounds for the operating range: + * lo: bw_lo, inflight_lo: conservative short-term lower bound + * hi: bw_hi, inflight_hi: robust long-term upper bound + * The bandwidth-probing time scale is (a) extended dynamically based on + * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by + * an interactive wall-clock time-scale to be more scalable and responsive + * than Reno and CUBIC. * * Here is a state transition diagram for BBR: * @@ -65,6 +66,13 @@ #include #include +#include +#include "tcp_dctcp.h" + +#define BBR_VERSION 3 + +#define bbr_param(sk,name) (bbr_ ## name) + /* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. @@ -85,36 +93,41 @@ enum bbr_mode { BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ }; +/* How does the incoming ACK stream relate to our bandwidth probing? */ +enum bbr_ack_phase { + BBR_ACKS_INIT, /* not probing; not getting probe feedback */ + BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ + BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ + BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ + BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ +}; + /* BBR congestion control block */ struct bbr { u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ - struct minmax bw; /* Max recent delivery rate in pkts/uS << 24 */ - u32 rtt_cnt; /* count of packet-timed rounds elapsed */ + u32 probe_rtt_min_us; /* min RTT in probe_rtt_win_ms win */ + u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ u64 cycle_mstamp; /* time of this cycle phase start */ - u32 mode:3, /* current bbr_mode in state machine */ + u32 mode:2, /* current bbr_mode in state machine */ prev_ca_state:3, /* CA state on previous ACK */ - packet_conservation:1, /* use packet conservation? */ round_start:1, /* start of packet-timed tx->ack round? */ + ce_state:1, /* If most recent data has CE bit set */ + bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ + try_fast_path:1, /* can we take fast path? */ idle_restart:1, /* restarting after idle? */ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ - unused:13, - lt_is_sampling:1, /* taking long-term ("LT") samples now? */ - lt_rtt_cnt:7, /* round trips in long-term interval */ - lt_use_bw:1; /* use lt_bw as our bw estimate? */ - u32 lt_bw; /* LT est delivery rate in pkts/uS << 24 */ - u32 lt_last_delivered; /* LT intvl start: tp->delivered */ - u32 lt_last_stamp; /* LT intvl start: tp->delivered_mstamp */ - u32 lt_last_lost; /* LT intvl start: tp->lost */ + init_cwnd:7, /* initial cwnd */ + unused_1:10; u32 pacing_gain:10, /* current gain for setting pacing rate */ cwnd_gain:10, /* current gain for setting cwnd */ full_bw_reached:1, /* reached full bw in Startup? */ full_bw_cnt:2, /* number of rounds without large bw gains */ - cycle_idx:3, /* current index in pacing_gain cycle array */ + cycle_idx:2, /* current index in pacing_gain cycle array */ has_seen_rtt:1, /* have we seen an RTT sample yet? */ - unused_b:5; + unused_2:6; u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ u32 full_bw; /* recent bw, to estimate if pipe is full */ @@ -124,19 +137,67 @@ struct bbr { u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ extra_acked_win_idx:1, /* current index in extra_acked array */ - unused_c:6; + /* BBR v3 state: */ + full_bw_now:1, /* recently reached full bw plateau? */ + startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ + loss_in_cycle:1, /* packet loss in this cycle? */ + ecn_in_cycle:1, /* ECN in this cycle? */ + unused_3:1; + u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ + u32 undo_bw_lo; /* bw_lo before latest losses */ + u32 undo_inflight_lo; /* inflight_lo before latest losses */ + u32 undo_inflight_hi; /* inflight_hi before latest losses */ + u32 bw_latest; /* max delivered bw in last round trip */ + u32 bw_lo; /* lower bound on sending bandwidth */ + u32 bw_hi[2]; /* max recent measured bw sample */ + u32 inflight_latest; /* max delivered data in last round trip */ + u32 inflight_lo; /* lower bound of inflight data range */ + u32 inflight_hi; /* upper bound of inflight data range */ + u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ + u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ + u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ + u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ + u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ + ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ + bw_probe_samples:1, /* rate samples reflect bw probing? */ + prev_probe_too_high:1, /* did last PROBE_UP go too high? */ + stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ + rounds_since_probe:8, /* packet-timed rounds since probed bw */ + loss_round_start:1, /* loss_round_delivered round trip? */ + loss_in_round:1, /* loss marked in this round trip? */ + ecn_in_round:1, /* ECN marked in this round trip? */ + ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ + loss_events_in_round:4,/* losses in STARTUP round */ + initialized:1; /* has bbr_init() been called? */ + u32 alpha_last_delivered; /* tp->delivered at alpha update */ + u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ + + u8 unused_4; /* to preserve alignment */ + struct tcp_plb_state plb; }; -#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ +struct bbr_context { + u32 sample_bw; +}; -/* Window length of bw filter (in rounds): */ -static const int bbr_bw_rtts = CYCLE_LEN + 2; /* Window length of min_rtt filter (in sec): */ static const u32 bbr_min_rtt_win_sec = 10; /* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode: */ static const u32 bbr_probe_rtt_mode_ms = 200; -/* Skip TSO below the following bandwidth (bits/sec): */ -static const int bbr_min_tso_rate = 1200000; +/* Window length of probe_rtt_min_us filter (in ms), and consequently the + * typical interval between PROBE_RTT mode entries. The default is 5000ms. + * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC + */ +static const u32 bbr_probe_rtt_win_ms = 5000; +/* Proportion of cwnd to estimated BDP in PROBE_RTT, in units of BBR_UNIT: */ +static const u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; + +/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting + * in bigger TSO bursts. We cut the RTT-based allowance in half + * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance + * is below 1500 bytes after 6 * ~500 usec = 3ms. + */ +static const u32 bbr_tso_rtt_shift = 9; /* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. * In order to help drive the network toward lower queues and low latency while @@ -146,13 +207,15 @@ static const int bbr_min_tso_rate = 1200000; */ static const int bbr_pacing_margin_percent = 1; -/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain +/* We use a startup_pacing_gain of 4*ln(2) because it's the smallest value * that will allow a smoothly increasing pacing rate that will double each RTT * and send the same number of packets per RTT that an un-paced, slow-starting * Reno or CUBIC flow would: */ -static const int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; -/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain +static const int bbr_startup_pacing_gain = BBR_UNIT * 277 / 100 + 1; +/* The gain for deriving startup cwnd: */ +static const int bbr_startup_cwnd_gain = BBR_UNIT * 2; +/* The pacing gain in BBR_DRAIN is calculated to typically drain * the queue created in BBR_STARTUP in a single round: */ static const int bbr_drain_gain = BBR_UNIT * 1000 / 2885; @@ -160,13 +223,17 @@ static const int bbr_drain_gain = BBR_UNIT * 1000 / 2885; static const int bbr_cwnd_gain = BBR_UNIT * 2; /* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw: */ static const int bbr_pacing_gain[] = { - BBR_UNIT * 5 / 4, /* probe for more available bw */ - BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ - BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ - BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ + BBR_UNIT * 5 / 4, /* UP: probe for more available bw */ + BBR_UNIT * 91 / 100, /* DOWN: drain queue and/or yield bw */ + BBR_UNIT, /* CRUISE: try to use pipe w/ some headroom */ + BBR_UNIT, /* REFILL: refill pipe to estimated 100% */ +}; +enum bbr_pacing_gain_phase { + BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ + BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ + BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ + BBR_BW_PROBE_REFILL = 3, /* refill the pipe again to 100% */ }; -/* Randomize the starting gain cycling phase over N phases: */ -static const u32 bbr_cycle_rand = 7; /* Try to keep at least this many packets in flight, if things go smoothly. For * smooth functioning, a sliding window protocol ACKing every other packet @@ -174,24 +241,12 @@ static const u32 bbr_cycle_rand = 7; */ static const u32 bbr_cwnd_min_target = 4; -/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ +/* To estimate if BBR_STARTUP or BBR_BW_PROBE_UP has filled pipe... */ /* If bw has increased significantly (1.25x), there may be more bw available: */ static const u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; /* But after 3 rounds w/o significant bw growth, estimate pipe is full: */ static const u32 bbr_full_bw_cnt = 3; -/* "long-term" ("LT") bandwidth estimator parameters... */ -/* The minimum number of rounds in an LT bw sampling interval: */ -static const u32 bbr_lt_intvl_min_rtts = 4; -/* If lost/delivered ratio > 20%, interval is "lossy" and we may be policed: */ -static const u32 bbr_lt_loss_thresh = 50; -/* If 2 intervals have a bw ratio <= 1/8, their bw is "consistent": */ -static const u32 bbr_lt_bw_ratio = BBR_UNIT / 8; -/* If 2 intervals have a bw diff <= 4 Kbit/sec their bw is "consistent": */ -static const u32 bbr_lt_bw_diff = 4000 / 8; -/* If we estimate we're policed, use lt_bw for this many round trips: */ -static const u32 bbr_lt_bw_max_rtts = 48; - /* Gain factor for adding extra_acked to target cwnd: */ static const int bbr_extra_acked_gain = BBR_UNIT; /* Window length of extra_acked window. */ @@ -201,8 +256,122 @@ static const u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; /* Time period for clamping cwnd increment due to ack aggregation */ static const u32 bbr_extra_acked_max_us = 100 * 1000; +/* Flags to control BBR ECN-related behavior... */ + +/* Ensure ACKs only ACK packets with consistent ECN CE status? */ +static const bool bbr_precise_ece_ack = true; + +/* Max RTT (in usec) at which to use sender-side ECN logic. + * Disabled when 0 (ECN allowed at any RTT). + */ +static const u32 bbr_ecn_max_rtt_us = 5000; + +/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. + * No loss response when 0. + */ +static const u32 bbr_beta = BBR_UNIT * 30 / 100; + +/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE (1/16 = 6.25%) */ +static const u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; + +/* The initial value for ecn_alpha; 1.0 allows a flow to respond quickly + * to congestion if the bottleneck is congested when the flow starts up. + */ +static const u32 bbr_ecn_alpha_init = BBR_UNIT; + +/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. + * No ECN based bounding when 0. + */ +static const u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ + +/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. + * Scaled by BBR_SCALE. Disabled when 0. + */ +static const u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ + +/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN + * clears then make the first round's increment to inflight_hi the following + * fraction of inflight_hi. + */ +static const u32 bbr_ecn_reprobe_gain = BBR_UNIT * 1 / 2; + +/* Estimate bw probing has gone too far if loss rate exceeds this level. */ +static const u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ + +/* Slow down for a packet loss recovered by TLP? */ +static const bool bbr_loss_probe_recovery = true; + +/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, + * and loss rate is higher than bbr_loss_thresh. + * Disabled if 0. + */ +static const u32 bbr_full_loss_cnt = 6; + +/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh + * meets this count. + */ +static const u32 bbr_full_ecn_cnt = 2; + +/* Fraction of unutilized headroom to try to leave in path upon high loss. */ +static const u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; + +/* How much do we increase cwnd_gain when probing for bandwidth in + * BBR_BW_PROBE_UP? This specifies the increment in units of + * BBR_UNIT/4. The default is 1, meaning 0.25. + * The min value is 0 (meaning 0.0); max is 3 (meaning 0.75). + */ +static const u32 bbr_bw_probe_cwnd_gain = 1; + +/* Max number of packet-timed rounds to wait before probing for bandwidth. If + * we want to tolerate 1% random loss per round, and not have this cut our + * inflight too much, we must probe for bw periodically on roughly this scale. + * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. + * We aim to be fair with Reno/CUBIC up to a BDP of at least: + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + */ +static const u32 bbr_bw_probe_max_rounds = 63; + +/* Max amount of randomness to inject in round counting for Reno-coexistence. + */ +static const u32 bbr_bw_probe_rand_rounds = 2; + +/* Use BBR-native probe time scale starting at this many usec. + * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: + * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs + */ +static const u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ + +/* Use BBR-native probes spread over this many usec: */ +static const u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ + +/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ +static const bool bbr_fast_path = true; + +/* Use fast ack mode? */ +static const bool bbr_fast_ack_mode = true; + +static u32 bbr_max_bw(const struct sock *sk); +static u32 bbr_bw(const struct sock *sk); +static void bbr_exit_probe_rtt(struct sock *sk); +static void bbr_reset_congestion_signals(struct sock *sk); +static void bbr_run_loss_probe_recovery(struct sock *sk); + static void bbr_check_probe_rtt_done(struct sock *sk); +/* This connection can use ECN if both endpoints have signaled ECN support in + * the handshake and the per-route settings indicated this is a + * shallow-threshold ECN environment, meaning both: + * (a) ECN CE marks indicate low-latency/shallow-threshold congestion, and + * (b) TCP endpoints provide precise ACKs that only ACK data segments + * with consistent ECN CE status + */ +static bool bbr_can_use_ecn(const struct sock *sk) +{ + const struct tcp_sock *tp = tcp_sk(sk); + + return (tcp_ecn_mode_any(tp)) && (tp->ecn_flags & TCP_ECN_LOW); +} + /* Do we estimate that STARTUP filled the pipe? */ static bool bbr_full_bw_reached(const struct sock *sk) { @@ -214,17 +383,17 @@ static bool bbr_full_bw_reached(const struct sock *sk) /* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ static u32 bbr_max_bw(const struct sock *sk) { - struct bbr *bbr = inet_csk_ca(sk); + const struct bbr *bbr = inet_csk_ca(sk); - return minmax_get(&bbr->bw); + return max(bbr->bw_hi[0], bbr->bw_hi[1]); } /* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ static u32 bbr_bw(const struct sock *sk) { - struct bbr *bbr = inet_csk_ca(sk); + const struct bbr *bbr = inet_csk_ca(sk); - return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk); + return min(bbr_max_bw(sk), bbr->bw_lo); } /* Return maximum extra acked in past k-2k round trips, @@ -241,15 +410,23 @@ static u16 bbr_extra_acked(const struct sock *sk) * The order here is chosen carefully to avoid overflow of u64. This should * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. */ -static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain) +static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, + int margin) { unsigned int mss = tcp_sk(sk)->mss_cache; rate *= mss; rate *= gain; rate >>= BBR_SCALE; - rate *= USEC_PER_SEC / 100 * (100 - bbr_pacing_margin_percent); - return rate >> BW_SCALE; + rate *= USEC_PER_SEC / 100 * (100 - margin); + rate >>= BW_SCALE; + rate = max(rate, 1ULL); + return rate; +} + +static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) +{ + return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); } /* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ @@ -257,12 +434,13 @@ static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) { u64 rate = bw; - rate = bbr_rate_bytes_per_sec(sk, rate, gain); + rate = bbr_rate_bytes_per_sec(sk, rate, gain, + bbr_pacing_margin_percent); rate = min_t(u64, rate, READ_ONCE(sk->sk_max_pacing_rate)); return rate; } -/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ +/* Initialize pacing rate to: startup_pacing_gain * init_cwnd / RTT. */ static void bbr_init_pacing_rate_from_rtt(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); @@ -279,7 +457,8 @@ static void bbr_init_pacing_rate_from_rtt(struct sock *sk) bw = (u64)tcp_snd_cwnd(tp) * BW_UNIT; do_div(bw, rtt_us); WRITE_ONCE(sk->sk_pacing_rate, - bbr_bw_to_pacing_rate(sk, bw, bbr_high_gain)); + bbr_bw_to_pacing_rate(sk, bw, + bbr_param(sk, startup_pacing_gain))); } /* Pace using current bw estimate and a gain factor. */ @@ -295,26 +474,48 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) WRITE_ONCE(sk->sk_pacing_rate, rate); } -/* override sysctl_tcp_min_tso_segs */ -__bpf_kfunc static u32 bbr_min_tso_segs(struct sock *sk) +/* Return the number of segments BBR would like in a TSO/GSO skb, given a + * particular max gso size as a constraint. TODO: make this simpler and more + * consistent by switching bbr to just call tcp_tso_autosize(). + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 segs, r; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift); + + /* Budget a TSO/GSO burst size allowance based on min_rtt. For every + * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. + * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) + */ + if (bbr_param(sk, tso_rtt_shift)) { + r = bbr->min_rtt_us >> bbr_param(sk, tso_rtt_shift); + if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ + bytes += GSO_LEGACY_MAX_SIZE >> r; + } + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +__bpf_kfunc static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) { - return READ_ONCE(sk->sk_pacing_rate) < (bbr_min_tso_rate >> 3) ? 1 : 2; + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); } +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ static u32 bbr_tso_segs_goal(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); - u32 segs, bytes; - - /* Sort of tcp_tso_autosize() but ignoring - * driver provided sk_gso_max_size. - */ - bytes = min_t(unsigned long, - READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift), - GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); - segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - return min(segs, 0x7FU); + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); } /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ @@ -334,7 +535,9 @@ __bpf_kfunc static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - if (event == CA_EVENT_TX_START && tp->app_limited) { + if (event == CA_EVENT_TX_START) { + if (!tp->app_limited) + return; bbr->idle_restart = 1; bbr->ack_epoch_mstamp = tp->tcp_mstamp; bbr->ack_epoch_acked = 0; @@ -345,6 +548,16 @@ __bpf_kfunc static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); else if (bbr->mode == BBR_PROBE_RTT) bbr_check_probe_rtt_done(sk); + } else if ((event == CA_EVENT_ECN_IS_CE || + event == CA_EVENT_ECN_NO_CE) && + bbr_can_use_ecn(sk) && + bbr_param(sk, precise_ece_ack)) { + u32 state = bbr->ce_state; + dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); + bbr->ce_state = state; + } else if (event == CA_EVENT_TLP_RECOVERY && + bbr_param(sk, loss_probe_recovery)) { + bbr_run_loss_probe_recovery(sk); } } @@ -367,10 +580,10 @@ static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) * default. This should only happen when the connection is not using TCP * timestamps and has retransmitted all of the SYN/SYNACK/data packets * ACKed so far. In this case, an RTO can cut cwnd to 1, in which - * case we need to slow-start up toward something safe: TCP_INIT_CWND. + * case we need to slow-start up toward something safe: initial cwnd. */ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ - return TCP_INIT_CWND; /* be safe: cap at default initial cwnd*/ + return bbr->init_cwnd; /* be safe: cap at initial cwnd */ w = (u64)bw * bbr->min_rtt_us; @@ -387,23 +600,23 @@ static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) * - one skb in sending host Qdisc, * - one skb in sending host TSO/GSO engine * - one skb being received by receiver host LRO/GRO/delayed-ACK engine - * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because - * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, + * Don't worry, at low rates this won't bloat cwnd because + * in such cases tso_segs_goal is small. The minimum cwnd is 4 packets, * which allows 2 outstanding 2-packet sequences, to try to keep pipe * full even with ACK-every-other-packet delayed ACKs. */ static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) { struct bbr *bbr = inet_csk_ca(sk); + u32 tso_segs_goal; - /* Allow enough full-sized skbs in flight to utilize end systems. */ - cwnd += 3 * bbr_tso_segs_goal(sk); - - /* Reduce delayed ACKs by rounding up cwnd to the next even number. */ - cwnd = (cwnd + 1) & ~1U; + tso_segs_goal = 3 * bbr_tso_segs_goal(sk); + /* Allow enough full-sized skbs in flight to utilize end systems. */ + cwnd = max_t(u32, cwnd, tso_segs_goal); + cwnd = max_t(u32, cwnd, bbr_param(sk, cwnd_min_target)); /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ - if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0) + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) cwnd += 2; return cwnd; @@ -458,10 +671,10 @@ static u32 bbr_ack_aggregation_cwnd(struct sock *sk) { u32 max_aggr_cwnd, aggr_cwnd = 0; - if (bbr_extra_acked_gain && bbr_full_bw_reached(sk)) { + if (bbr_param(sk, extra_acked_gain)) { max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) / BW_UNIT; - aggr_cwnd = (bbr_extra_acked_gain * bbr_extra_acked(sk)) + aggr_cwnd = (bbr_param(sk, extra_acked_gain) * bbr_extra_acked(sk)) >> BBR_SCALE; aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); } @@ -469,66 +682,27 @@ static u32 bbr_ack_aggregation_cwnd(struct sock *sk) return aggr_cwnd; } -/* An optimization in BBR to reduce losses: On the first round of recovery, we - * follow the packet conservation principle: send P packets per P packets acked. - * After that, we slow-start and send at most 2*P packets per P packets acked. - * After recovery finishes, or upon undo, we restore the cwnd we had when - * recovery started (capped by the target cwnd based on estimated BDP). - * - * TODO(ycheng/ncardwell): implement a rate-based approach. - */ -static bool bbr_set_cwnd_to_recover_or_restore( - struct sock *sk, const struct rate_sample *rs, u32 acked, u32 *new_cwnd) +/* Returns the cwnd for PROBE_RTT mode. */ +static u32 bbr_probe_rtt_cwnd(struct sock *sk) { - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state; - u32 cwnd = tcp_snd_cwnd(tp); - - /* An ACK for P pkts should release at most 2*P packets. We do this - * in two steps. First, here we deduct the number of lost packets. - * Then, in bbr_set_cwnd() we slow start up toward the target cwnd. - */ - if (rs->losses > 0) - cwnd = max_t(s32, cwnd - rs->losses, 1); - - if (state == TCP_CA_Recovery && prev_state != TCP_CA_Recovery) { - /* Starting 1st round of Recovery, so do packet conservation. */ - bbr->packet_conservation = 1; - bbr->next_rtt_delivered = tp->delivered; /* start round now */ - /* Cut unused cwnd from app behavior, TSQ, or TSO deferral: */ - cwnd = tcp_packets_in_flight(tp) + acked; - } else if (prev_state >= TCP_CA_Recovery && state < TCP_CA_Recovery) { - /* Exiting loss recovery; restore cwnd saved before recovery. */ - cwnd = max(cwnd, bbr->prior_cwnd); - bbr->packet_conservation = 0; - } - bbr->prev_ca_state = state; - - if (bbr->packet_conservation) { - *new_cwnd = max(cwnd, tcp_packets_in_flight(tp) + acked); - return true; /* yes, using packet conservation */ - } - *new_cwnd = cwnd; - return false; + return max_t(u32, bbr_param(sk, cwnd_min_target), + bbr_bdp(sk, bbr_bw(sk), bbr_param(sk, probe_rtt_cwnd_gain))); } /* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss * has drawn us down below target), or snap down to target if we're above it. */ static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, - u32 acked, u32 bw, int gain) + u32 acked, u32 bw, int gain, u32 cwnd, + struct bbr_context *ctx) { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u32 cwnd = tcp_snd_cwnd(tp), target_cwnd = 0; + u32 target_cwnd = 0; if (!acked) goto done; /* no packet fully ACKed; just apply caps */ - if (bbr_set_cwnd_to_recover_or_restore(sk, rs, acked, &cwnd)) - goto done; - target_cwnd = bbr_bdp(sk, bw, gain); /* Increment the cwnd to account for excess ACKed data that seems @@ -537,74 +711,26 @@ static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, target_cwnd += bbr_ack_aggregation_cwnd(sk); target_cwnd = bbr_quantization_budget(sk, target_cwnd); - /* If we're below target cwnd, slow start cwnd toward target cwnd. */ - if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ - cwnd = min(cwnd + acked, target_cwnd); - else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND) - cwnd = cwnd + acked; - cwnd = max(cwnd, bbr_cwnd_min_target); + /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ + bbr->try_fast_path = 0; + if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ + cwnd += acked; + if (cwnd >= target_cwnd) { + cwnd = target_cwnd; + bbr->try_fast_path = 1; + } + } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { + cwnd += acked; + } else { + bbr->try_fast_path = 1; + } + cwnd = max_t(u32, cwnd, bbr_param(sk, cwnd_min_target)); done: - tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); /* apply global cap */ + tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); /* global cap */ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ - tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), bbr_cwnd_min_target)); -} - -/* End cycle phase if it's time and/or we hit the phase's in-flight target. */ -static bool bbr_is_next_cycle_phase(struct sock *sk, - const struct rate_sample *rs) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - bool is_full_length = - tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp) > - bbr->min_rtt_us; - u32 inflight, bw; - - /* The pacing_gain of 1.0 paces at the estimated bw to try to fully - * use the pipe without increasing the queue. - */ - if (bbr->pacing_gain == BBR_UNIT) - return is_full_length; /* just use wall clock time */ - - inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); - bw = bbr_max_bw(sk); - - /* A pacing_gain > 1.0 probes for bw by trying to raise inflight to at - * least pacing_gain*BDP; this may take more than min_rtt if min_rtt is - * small (e.g. on a LAN). We do not persist if packets are lost, since - * a path with small buffers may not hold that much. - */ - if (bbr->pacing_gain > BBR_UNIT) - return is_full_length && - (rs->losses || /* perhaps pacing_gain*BDP won't fit */ - inflight >= bbr_inflight(sk, bw, bbr->pacing_gain)); - - /* A pacing_gain < 1.0 tries to drain extra queue we added if bw - * probing didn't find more bw. If inflight falls to match BDP then we - * estimate queue is drained; persisting would underutilize the pipe. - */ - return is_full_length || - inflight <= bbr_inflight(sk, bw, BBR_UNIT); -} - -static void bbr_advance_cycle_phase(struct sock *sk) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - - bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1); - bbr->cycle_mstamp = tp->delivered_mstamp; -} - -/* Gain cycling: cycle pacing gain to converge to fair share of available bw. */ -static void bbr_update_cycle_phase(struct sock *sk, - const struct rate_sample *rs) -{ - struct bbr *bbr = inet_csk_ca(sk); - - if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs)) - bbr_advance_cycle_phase(sk); + tcp_snd_cwnd_set(tp, min_t(u32, tcp_snd_cwnd(tp), + bbr_probe_rtt_cwnd(sk))); } static void bbr_reset_startup_mode(struct sock *sk) @@ -614,191 +740,49 @@ static void bbr_reset_startup_mode(struct sock *sk) bbr->mode = BBR_STARTUP; } -static void bbr_reset_probe_bw_mode(struct sock *sk) -{ - struct bbr *bbr = inet_csk_ca(sk); - - bbr->mode = BBR_PROBE_BW; - bbr->cycle_idx = CYCLE_LEN - 1 - get_random_u32_below(bbr_cycle_rand); - bbr_advance_cycle_phase(sk); /* flip to next phase of gain cycle */ -} - -static void bbr_reset_mode(struct sock *sk) -{ - if (!bbr_full_bw_reached(sk)) - bbr_reset_startup_mode(sk); - else - bbr_reset_probe_bw_mode(sk); -} - -/* Start a new long-term sampling interval. */ -static void bbr_reset_lt_bw_sampling_interval(struct sock *sk) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - - bbr->lt_last_stamp = div_u64(tp->delivered_mstamp, USEC_PER_MSEC); - bbr->lt_last_delivered = tp->delivered; - bbr->lt_last_lost = tp->lost; - bbr->lt_rtt_cnt = 0; -} - -/* Completely reset long-term bandwidth sampling. */ -static void bbr_reset_lt_bw_sampling(struct sock *sk) -{ - struct bbr *bbr = inet_csk_ca(sk); - - bbr->lt_bw = 0; - bbr->lt_use_bw = 0; - bbr->lt_is_sampling = false; - bbr_reset_lt_bw_sampling_interval(sk); -} - -/* Long-term bw sampling interval is done. Estimate whether we're policed. */ -static void bbr_lt_bw_interval_done(struct sock *sk, u32 bw) -{ - struct bbr *bbr = inet_csk_ca(sk); - u32 diff; - - if (bbr->lt_bw) { /* do we have bw from a previous interval? */ - /* Is new bw close to the lt_bw from the previous interval? */ - diff = abs(bw - bbr->lt_bw); - if ((diff * BBR_UNIT <= bbr_lt_bw_ratio * bbr->lt_bw) || - (bbr_rate_bytes_per_sec(sk, diff, BBR_UNIT) <= - bbr_lt_bw_diff)) { - /* All criteria are met; estimate we're policed. */ - bbr->lt_bw = (bw + bbr->lt_bw) >> 1; /* avg 2 intvls */ - bbr->lt_use_bw = 1; - bbr->pacing_gain = BBR_UNIT; /* try to avoid drops */ - bbr->lt_rtt_cnt = 0; - return; - } - } - bbr->lt_bw = bw; - bbr_reset_lt_bw_sampling_interval(sk); -} - -/* Token-bucket traffic policers are common (see "An Internet-Wide Analysis of - * Traffic Policing", SIGCOMM 2016). BBR detects token-bucket policers and - * explicitly models their policed rate, to reduce unnecessary losses. We - * estimate that we're policed if we see 2 consecutive sampling intervals with - * consistent throughput and high packet loss. If we think we're being policed, - * set lt_bw to the "long-term" average delivery rate from those 2 intervals. +/* See if we have reached next round trip. Upon start of the new round, + * returns packets delivered since previous round start plus this ACK. */ -static void bbr_lt_bw_sampling(struct sock *sk, const struct rate_sample *rs) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - u32 lost, delivered; - u64 bw; - u32 t; - - if (bbr->lt_use_bw) { /* already using long-term rate, lt_bw? */ - if (bbr->mode == BBR_PROBE_BW && bbr->round_start && - ++bbr->lt_rtt_cnt >= bbr_lt_bw_max_rtts) { - bbr_reset_lt_bw_sampling(sk); /* stop using lt_bw */ - bbr_reset_probe_bw_mode(sk); /* restart gain cycling */ - } - return; - } - - /* Wait for the first loss before sampling, to let the policer exhaust - * its tokens and estimate the steady-state rate allowed by the policer. - * Starting samples earlier includes bursts that over-estimate the bw. - */ - if (!bbr->lt_is_sampling) { - if (!rs->losses) - return; - bbr_reset_lt_bw_sampling_interval(sk); - bbr->lt_is_sampling = true; - } - - /* To avoid underestimates, reset sampling if we run out of data. */ - if (rs->is_app_limited) { - bbr_reset_lt_bw_sampling(sk); - return; - } - - if (bbr->round_start) - bbr->lt_rtt_cnt++; /* count round trips in this interval */ - if (bbr->lt_rtt_cnt < bbr_lt_intvl_min_rtts) - return; /* sampling interval needs to be longer */ - if (bbr->lt_rtt_cnt > 4 * bbr_lt_intvl_min_rtts) { - bbr_reset_lt_bw_sampling(sk); /* interval is too long */ - return; - } - - /* End sampling interval when a packet is lost, so we estimate the - * policer tokens were exhausted. Stopping the sampling before the - * tokens are exhausted under-estimates the policed rate. - */ - if (!rs->losses) - return; - - /* Calculate packets lost and delivered in sampling interval. */ - lost = tp->lost - bbr->lt_last_lost; - delivered = tp->delivered - bbr->lt_last_delivered; - /* Is loss rate (lost/delivered) >= lt_loss_thresh? If not, wait. */ - if (!delivered || (lost << BBR_SCALE) < bbr_lt_loss_thresh * delivered) - return; - - /* Find average delivery rate in this sampling interval. */ - t = div_u64(tp->delivered_mstamp, USEC_PER_MSEC) - bbr->lt_last_stamp; - if ((s32)t < 1) - return; /* interval is less than one ms, so wait */ - /* Check if can multiply without overflow */ - if (t >= ~0U / USEC_PER_MSEC) { - bbr_reset_lt_bw_sampling(sk); /* interval too long; reset */ - return; - } - t *= USEC_PER_MSEC; - bw = (u64)delivered * BW_UNIT; - do_div(bw, t); - bbr_lt_bw_interval_done(sk, bw); -} - -/* Estimate the bandwidth based on how fast packets are delivered */ -static void bbr_update_bw(struct sock *sk, const struct rate_sample *rs) +static u32 bbr_update_round_start(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u64 bw; + u32 round_delivered = 0; bbr->round_start = 0; - if (rs->delivered < 0 || rs->interval_us <= 0) - return; /* Not a valid observation */ /* See if we've reached the next RTT */ - if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) { + if (rs->interval_us > 0 && + !before(rs->prior_delivered, bbr->next_rtt_delivered)) { + round_delivered = tp->delivered - bbr->next_rtt_delivered; bbr->next_rtt_delivered = tp->delivered; - bbr->rtt_cnt++; bbr->round_start = 1; - bbr->packet_conservation = 0; } + return round_delivered; +} - bbr_lt_bw_sampling(sk, rs); +/* Calculate the bandwidth based on how fast packets are delivered */ +static void bbr_calculate_bw_sample(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + u64 bw = 0; /* Divide delivered by the interval to find a (lower bound) bottleneck * bandwidth sample. Delivered is in packets and interval_us in uS and * ratio will be <<1 for most connections. So delivered is first scaled. + * Round up to allow growth at low rates, even with integer division. */ - bw = div64_long((u64)rs->delivered * BW_UNIT, rs->interval_us); - - /* If this sample is application-limited, it is likely to have a very - * low delivered count that represents application behavior rather than - * the available network rate. Such a sample could drag down estimated - * bw, causing needless slow-down. Thus, to continue to send at the - * last measured network rate, we filter out app-limited samples unless - * they describe the path bw at least as well as our bw model. - * - * So the goal during app-limited phase is to proceed with the best - * network rate no matter how long. We automatically leave this - * phase when app writes faster than the network can deliver :) - */ - if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) { - /* Incorporate new sample into our max bw filter. */ - minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw); + if (rs->interval_us > 0) { + if (WARN_ONCE(rs->delivered < 0, + "negative delivered: %d interval_us: %ld\n", + rs->delivered, rs->interval_us)) + return; + + bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); } + + ctx->sample_bw = bw; } /* Estimates the windowed max degree of ack aggregation. @@ -812,7 +796,7 @@ static void bbr_update_bw(struct sock *sk, const struct rate_sample *rs) * * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). * Max filter is an approximate sliding window of 5-10 (packet timed) round - * trips. + * trips for non-startup phase, and 1-2 round trips for startup. */ static void bbr_update_ack_aggregation(struct sock *sk, const struct rate_sample *rs) @@ -820,15 +804,19 @@ static void bbr_update_ack_aggregation(struct sock *sk, u32 epoch_us, expected_acked, extra_acked; struct bbr *bbr = inet_csk_ca(sk); struct tcp_sock *tp = tcp_sk(sk); + u32 extra_acked_win_rtts_thresh = bbr_param(sk, extra_acked_win_rtts); - if (!bbr_extra_acked_gain || rs->acked_sacked <= 0 || + if (!bbr_param(sk, extra_acked_gain) || rs->acked_sacked <= 0 || rs->delivered < 0 || rs->interval_us <= 0) return; if (bbr->round_start) { bbr->extra_acked_win_rtts = min(0x1F, bbr->extra_acked_win_rtts + 1); - if (bbr->extra_acked_win_rtts >= bbr_extra_acked_win_rtts) { + if (!bbr_full_bw_reached(sk)) + extra_acked_win_rtts_thresh = 1; + if (bbr->extra_acked_win_rtts >= + extra_acked_win_rtts_thresh) { bbr->extra_acked_win_rtts = 0; bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? 0 : 1; @@ -862,49 +850,6 @@ static void bbr_update_ack_aggregation(struct sock *sk, bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; } -/* Estimate when the pipe is full, using the change in delivery rate: BBR - * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by - * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited - * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the - * higher rwin, 3: we get higher delivery rate samples. Or transient - * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar - * design goal, but uses delay and inter-ACK spacing instead of bandwidth. - */ -static void bbr_check_full_bw_reached(struct sock *sk, - const struct rate_sample *rs) -{ - struct bbr *bbr = inet_csk_ca(sk); - u32 bw_thresh; - - if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) - return; - - bw_thresh = (u64)bbr->full_bw * bbr_full_bw_thresh >> BBR_SCALE; - if (bbr_max_bw(sk) >= bw_thresh) { - bbr->full_bw = bbr_max_bw(sk); - bbr->full_bw_cnt = 0; - return; - } - ++bbr->full_bw_cnt; - bbr->full_bw_reached = bbr->full_bw_cnt >= bbr_full_bw_cnt; -} - -/* If pipe is probably full, drain the queue and then enter steady-state. */ -static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs) -{ - struct bbr *bbr = inet_csk_ca(sk); - - if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { - bbr->mode = BBR_DRAIN; /* drain queue we created */ - tcp_sk(sk)->snd_ssthresh = - bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); - } /* fall through to check if in-flight is already small: */ - if (bbr->mode == BBR_DRAIN && - bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= - bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) - bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ -} - static void bbr_check_probe_rtt_done(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); @@ -914,9 +859,9 @@ static void bbr_check_probe_rtt_done(struct sock *sk) after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) return; - bbr->min_rtt_stamp = tcp_jiffies32; /* wait a while until PROBE_RTT */ + bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); - bbr_reset_mode(sk); + bbr_exit_probe_rtt(sk); } /* The goal of PROBE_RTT mode is to have BBR flows cooperatively and @@ -942,23 +887,35 @@ static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - bool filter_expired; + bool probe_rtt_expired, min_rtt_expired; + u32 expire; - /* Track min RTT seen in the min_rtt_win_sec filter window: */ - filter_expired = after(tcp_jiffies32, - bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ); + /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ + expire = bbr->probe_rtt_min_stamp + + msecs_to_jiffies(bbr_param(sk, probe_rtt_win_ms)); + probe_rtt_expired = after(tcp_jiffies32, expire); if (rs->rtt_us >= 0 && - (rs->rtt_us < bbr->min_rtt_us || - (filter_expired && !rs->is_ack_delayed))) { - bbr->min_rtt_us = rs->rtt_us; - bbr->min_rtt_stamp = tcp_jiffies32; + (rs->rtt_us < bbr->probe_rtt_min_us || + (probe_rtt_expired && !rs->is_ack_delayed))) { + bbr->probe_rtt_min_us = rs->rtt_us; + bbr->probe_rtt_min_stamp = tcp_jiffies32; + } + /* Track min RTT seen in the min_rtt_win_sec filter window: */ + expire = bbr->min_rtt_stamp + bbr_param(sk, min_rtt_win_sec) * HZ; + min_rtt_expired = after(tcp_jiffies32, expire); + if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || + min_rtt_expired) { + bbr->min_rtt_us = bbr->probe_rtt_min_us; + bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; } - if (bbr_probe_rtt_mode_ms > 0 && filter_expired && + if (bbr_param(sk, probe_rtt_mode_ms) > 0 && probe_rtt_expired && !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ bbr->probe_rtt_done_stamp = 0; + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; } if (bbr->mode == BBR_PROBE_RTT) { @@ -967,9 +924,9 @@ static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; /* Maintain min packets in flight for max(200 ms, 1 round). */ if (!bbr->probe_rtt_done_stamp && - tcp_packets_in_flight(tp) <= bbr_cwnd_min_target) { + tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { bbr->probe_rtt_done_stamp = tcp_jiffies32 + - msecs_to_jiffies(bbr_probe_rtt_mode_ms); + msecs_to_jiffies(bbr_param(sk, probe_rtt_mode_ms)); bbr->probe_rtt_round_done = 0; bbr->next_rtt_delivered = tp->delivered; } else if (bbr->probe_rtt_done_stamp) { @@ -990,18 +947,20 @@ static void bbr_update_gains(struct sock *sk) switch (bbr->mode) { case BBR_STARTUP: - bbr->pacing_gain = bbr_high_gain; - bbr->cwnd_gain = bbr_high_gain; + bbr->pacing_gain = bbr_param(sk, startup_pacing_gain); + bbr->cwnd_gain = bbr_param(sk, startup_cwnd_gain); break; case BBR_DRAIN: - bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */ - bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */ + bbr->pacing_gain = bbr_param(sk, drain_gain); /* slow, to drain */ + bbr->cwnd_gain = bbr_param(sk, startup_cwnd_gain); /* keep cwnd */ break; case BBR_PROBE_BW: - bbr->pacing_gain = (bbr->lt_use_bw ? - BBR_UNIT : - bbr_pacing_gain[bbr->cycle_idx]); - bbr->cwnd_gain = bbr_cwnd_gain; + bbr->pacing_gain = bbr_pacing_gain[bbr->cycle_idx]; + bbr->cwnd_gain = bbr_param(sk, cwnd_gain); + if (bbr_param(sk, bw_probe_cwnd_gain) && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr->cwnd_gain += + BBR_UNIT * bbr_param(sk, bw_probe_cwnd_gain) / 4; break; case BBR_PROBE_RTT: bbr->pacing_gain = BBR_UNIT; @@ -1013,144 +972,1387 @@ static void bbr_update_gains(struct sock *sk) } } -static void bbr_update_model(struct sock *sk, const struct rate_sample *rs) +__bpf_kfunc static u32 bbr_sndbuf_expand(struct sock *sk) { - bbr_update_bw(sk, rs); - bbr_update_ack_aggregation(sk, rs); - bbr_update_cycle_phase(sk, rs); - bbr_check_full_bw_reached(sk, rs); - bbr_check_drain(sk, rs); - bbr_update_min_rtt(sk, rs); - bbr_update_gains(sk); + /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ + return 3; } -__bpf_kfunc static void bbr_main(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs) +/* Incorporate a new bw sample into the current window of our max filter. */ +static void bbr_take_max_bw_sample(struct sock *sk, u32 bw) { struct bbr *bbr = inet_csk_ca(sk); - u32 bw; - - bbr_update_model(sk, rs); - bw = bbr_bw(sk); - bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); - bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain); + bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); } -__bpf_kfunc static void bbr_init(struct sock *sk) +/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ +static void bbr_advance_max_bw_filter(struct sock *sk) { - struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - bbr->prior_cwnd = 0; - tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; - bbr->rtt_cnt = 0; - bbr->next_rtt_delivered = tp->delivered; - bbr->prev_ca_state = TCP_CA_Open; - bbr->packet_conservation = 0; - - bbr->probe_rtt_done_stamp = 0; - bbr->probe_rtt_round_done = 0; - bbr->min_rtt_us = tcp_min_rtt(tp); - bbr->min_rtt_stamp = tcp_jiffies32; - - minmax_reset(&bbr->bw, bbr->rtt_cnt, 0); /* init max bw to 0 */ + if (!bbr->bw_hi[1]) + return; /* no samples in this window; remember old window */ + bbr->bw_hi[0] = bbr->bw_hi[1]; + bbr->bw_hi[1] = 0; +} - bbr->has_seen_rtt = 0; - bbr_init_pacing_rate_from_rtt(sk); +/* Reset the estimator for reaching full bandwidth based on bw plateau. */ +static void bbr_reset_full_bw(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); - bbr->round_start = 0; - bbr->idle_restart = 0; - bbr->full_bw_reached = 0; bbr->full_bw = 0; bbr->full_bw_cnt = 0; - bbr->cycle_mstamp = 0; - bbr->cycle_idx = 0; - bbr_reset_lt_bw_sampling(sk); - bbr_reset_startup_mode(sk); + bbr->full_bw_now = 0; +} - bbr->ack_epoch_mstamp = tp->tcp_mstamp; - bbr->ack_epoch_acked = 0; - bbr->extra_acked_win_rtts = 0; - bbr->extra_acked_win_idx = 0; - bbr->extra_acked[0] = 0; - bbr->extra_acked[1] = 0; +/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ +static u32 bbr_target_inflight(struct sock *sk) +{ + u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); - cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + return min(bdp, tcp_sk(sk)->snd_cwnd); } -__bpf_kfunc static u32 bbr_sndbuf_expand(struct sock *sk) +static bool bbr_is_probing_bandwidth(struct sock *sk) { - /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ - return 3; + struct bbr *bbr = inet_csk_ca(sk); + + return (bbr->mode == BBR_STARTUP) || + (bbr->mode == BBR_PROBE_BW && + (bbr->cycle_idx == BBR_BW_PROBE_REFILL || + bbr->cycle_idx == BBR_BW_PROBE_UP)); +} + +/* Has the given amount of time elapsed since we marked the phase start? */ +static bool bbr_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) +{ + const struct tcp_sock *tp = tcp_sk(sk); + const struct bbr *bbr = inet_csk_ca(sk); + + return tcp_stamp_us_delta(tp->tcp_mstamp, + bbr->cycle_mstamp + interval_us) > 0; +} + +static void bbr_handle_queue_too_high_in_startup(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bdp; /* estimated BDP in packets, with quantization budget */ + + bbr->full_bw_reached = 1; + + bdp = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr->inflight_hi = max(bdp, bbr->inflight_latest); +} + +/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ +static void bbr_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || + !bbr_param(sk, full_ecn_cnt) || !bbr_param(sk, ecn_thresh)) + return; + + if (ce_ratio >= bbr_param(sk, ecn_thresh)) + bbr->startup_ecn_rounds++; + else + bbr->startup_ecn_rounds = 0; + + if (bbr->startup_ecn_rounds >= bbr_param(sk, full_ecn_cnt)) { + bbr_handle_queue_too_high_in_startup(sk); + return; + } +} + +/* Updates ecn_alpha and returns ce_ratio. -1 if not available. */ +static int bbr_update_ecn_alpha(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct net *net = sock_net(sk); + struct bbr *bbr = inet_csk_ca(sk); + s32 delivered, delivered_ce; + u64 alpha, ce_ratio; + u32 gain; + bool want_ecn_alpha; + + /* See if we should use ECN sender logic for this connection. */ + if (!bbr->ecn_eligible && bbr_can_use_ecn(sk) && + !!bbr_param(sk, ecn_factor) && + (bbr->min_rtt_us <= bbr_ecn_max_rtt_us || + !bbr_ecn_max_rtt_us)) + bbr->ecn_eligible = 1; + + /* Skip updating alpha only if not ECN-eligible and PLB is disabled. */ + want_ecn_alpha = (bbr->ecn_eligible || + (bbr_can_use_ecn(sk) && + READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))); + if (!want_ecn_alpha) + return -1; + + delivered = tp->delivered - bbr->alpha_last_delivered; + delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; + + if (delivered == 0 || /* avoid divide by zero */ + WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ + return -1; + + BUILD_BUG_ON(BBR_SCALE != TCP_PLB_SCALE); + ce_ratio = (u64)delivered_ce << BBR_SCALE; + do_div(ce_ratio, delivered); + + gain = bbr_param(sk, ecn_alpha_gain); + alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; + alpha += (gain * ce_ratio) >> BBR_SCALE; + bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); + + bbr->alpha_last_delivered = tp->delivered; + bbr->alpha_last_delivered_ce = tp->delivered_ce; + + bbr_check_ecn_too_high_in_startup(sk, ce_ratio); + return (int)ce_ratio; } -/* In theory BBR does not need to undo the cwnd since it does not - * always reduce cwnd on losses (see bbr_main()). Keep it for now. +/* Protective Load Balancing (PLB). PLB rehashes outgoing data (to a new IPv6 + * flow label) if it encounters sustained congestion in the form of ECN marks. */ -__bpf_kfunc static u32 bbr_undo_cwnd(struct sock *sk) +static void bbr_plb(struct sock *sk, const struct rate_sample *rs, int ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->round_start && ce_ratio >= 0) + tcp_plb_update_state(sk, &bbr->plb, ce_ratio); + + tcp_plb_check_rehash(sk, &bbr->plb); +} + +/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ +static void bbr_raise_inflight_hi_slope(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 growth_this_round, cnt; + + /* Calculate "slope": packets S/Acked per inflight_hi increment. */ + growth_this_round = 1 << bbr->bw_probe_up_rounds; + bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); + cnt = tcp_snd_cwnd(tp) / growth_this_round; + cnt = max(cnt, 1U); + bbr->bw_probe_up_cnt = cnt; +} + +/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ +static void bbr_probe_inflight_hi_upward(struct sock *sk, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 delta; + + if (!tp->is_cwnd_limited || tcp_snd_cwnd(tp) < bbr->inflight_hi) + return; /* not fully using inflight_hi, so don't grow it */ + + /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ + bbr->bw_probe_up_acks += rs->acked_sacked; + if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { + delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; + bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; + bbr->inflight_hi += delta; + bbr->try_fast_path = 0; /* Need to update cwnd */ + } + + if (bbr->round_start) + bbr_raise_inflight_hi_slope(sk); +} + +/* Does loss/ECN rate for this sample say inflight is "too high"? + * This is used by both the bbr_check_loss_too_high_in_startup() function, + * and in PROBE_UP. + */ +static bool bbr_is_inflight_too_high(const struct sock *sk, + const struct rate_sample *rs) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh, ecn_thresh; + + if (rs->lost > 0 && rs->tx_in_flight) { + loss_thresh = (u64)rs->tx_in_flight * bbr_param(sk, loss_thresh) >> + BBR_SCALE; + if (rs->lost > loss_thresh) { + return true; + } + } + + if (rs->delivered_ce > 0 && rs->delivered > 0 && + bbr->ecn_eligible && !!bbr_param(sk, ecn_thresh)) { + ecn_thresh = (u64)rs->delivered * bbr_param(sk, ecn_thresh) >> + BBR_SCALE; + if (rs->delivered_ce > ecn_thresh) { + return true; + } + } + + return false; +} + +/* Calculate the tx_in_flight level that corresponded to excessive loss. + * We find "lost_prefix" segs of the skb where loss rate went too high, + * by solving for "lost_prefix" in the following equation: + * lost / inflight >= loss_thresh + * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh + * Then we take that equation, convert it to fixed point, and + * round up to the nearest packet. + */ +static u32 bbr_inflight_hi_from_lost_skb(const struct sock *sk, + const struct rate_sample *rs, + const struct sk_buff *skb) +{ + const struct tcp_sock *tp = tcp_sk(sk); + u32 loss_thresh = bbr_param(sk, loss_thresh); + u32 pcount, divisor, inflight_hi; + s32 inflight_prev, lost_prev; + u64 loss_budget, lost_prefix; + + pcount = tcp_skb_pcount(skb); + + /* How much data was in flight before this skb? */ + inflight_prev = rs->tx_in_flight - pcount; + if (inflight_prev < 0) { + WARN_ONCE(tcp_skb_tx_in_flight_is_suspicious( + pcount, + TCP_SKB_CB(skb)->sacked, + rs->tx_in_flight), + "tx_in_flight: %u pcount: %u reneg: %u", + rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg); + return ~0U; + } + + /* How much inflight data was marked lost before this skb? */ + lost_prev = rs->lost - pcount; + if (WARN_ONCE(lost_prev < 0, + "cwnd: %u ca: %d out: %u lost: %u pif: %u " + "tx_in_flight: %u tx.lost: %u tp->lost: %u rs->lost: %d " + "lost_prev: %d pcount: %d seq: %u end_seq: %u reneg: %u", + tcp_snd_cwnd(tp), inet_csk(sk)->icsk_ca_state, + tp->packets_out, tp->lost_out, tcp_packets_in_flight(tp), + rs->tx_in_flight, TCP_SKB_CB(skb)->tx.lost, tp->lost, + rs->lost, lost_prev, pcount, + TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq, + tp->is_sack_reneg)) + return ~0U; + + /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ + loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; + loss_budget >>= BBR_SCALE; + if (lost_prev >= loss_budget) { + lost_prefix = 0; /* previous losses crossed loss_thresh */ + } else { + lost_prefix = loss_budget - lost_prev; + lost_prefix <<= BBR_SCALE; + divisor = BBR_UNIT - loss_thresh; + if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ + return ~0U; + do_div(lost_prefix, divisor); + } + + inflight_hi = inflight_prev + lost_prefix; + return inflight_hi; +} + +/* If loss/ECN rates during probing indicated we may have overfilled a + * buffer, return an operating point that tries to leave unutilized headroom in + * the path for other flows, for fairness convergence and lower RTTs and loss. + */ +static u32 bbr_inflight_with_headroom(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 headroom, headroom_fraction; + + if (bbr->inflight_hi == ~0U) + return ~0U; + + headroom_fraction = bbr_param(sk, inflight_headroom); + headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; + headroom = max(headroom, 1U); + return max_t(s32, bbr->inflight_hi - headroom, + bbr_param(sk, cwnd_min_target)); +} + +/* Bound cwnd to a sensible level, based on our current probing state + * machine phase and model of a good inflight level (inflight_lo, inflight_hi). + */ +static void bbr_bound_cwnd_for_inflight_model(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 cap; + + /* tcp_rcv_synsent_state_process() currently calls tcp_ack() + * and thus cong_control() without first initializing us(!). + */ + if (!bbr->initialized) + return; + + cap = ~0U; + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { + /* Probe to see if more packets fit in the path. */ + cap = bbr->inflight_hi; + } else { + if (bbr->mode == BBR_PROBE_RTT || + (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) + cap = bbr_inflight_with_headroom(sk); + } + /* Adapt to any loss/ECN since our last bw probe. */ + cap = min(cap, bbr->inflight_lo); + + cap = max_t(u32, cap, bbr_param(sk, cwnd_min_target)); + tcp_snd_cwnd_set(tp, min(cap, tcp_snd_cwnd(tp))); +} + +/* How should we multiplicatively cut bw or inflight limits based on ECN? */ +static u32 bbr_ecn_cut(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return BBR_UNIT - + ((bbr->ecn_alpha * bbr_param(sk, ecn_factor)) >> BBR_SCALE); +} + +/* Init lower bounds if have not inited yet. */ +static void bbr_init_lower_bounds(struct sock *sk, bool init_bw) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (init_bw && bbr->bw_lo == ~0U) + bbr->bw_lo = bbr_max_bw(sk); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tcp_snd_cwnd(tp); +} + +/* Reduce bw and inflight to (1 - beta). */ +static void bbr_loss_lower_bounds(struct sock *sk, u32 *bw, u32 *inflight) +{ + struct bbr* bbr = inet_csk_ca(sk); + u32 loss_cut = BBR_UNIT - bbr_param(sk, beta); + + *bw = max_t(u32, bbr->bw_latest, + (u64)bbr->bw_lo * loss_cut >> BBR_SCALE); + *inflight = max_t(u32, bbr->inflight_latest, + (u64)bbr->inflight_lo * loss_cut >> BBR_SCALE); +} + +/* Reduce inflight to (1 - alpha*ecn_factor). */ +static void bbr_ecn_lower_bounds(struct sock *sk, u32 *inflight) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_cut = bbr_ecn_cut(sk); + + *inflight = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; +} + +/* Estimate a short-term lower bound on the capacity available now, based + * on measurements of the current delivery process and recent history. When we + * are seeing loss/ECN at times when we are not probing bw, then conservatively + * move toward flow balance by multiplicatively cutting our short-term + * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a + * multiplicative decrease in order to converge to a lower capacity in time + * logarithmic in the magnitude of the decrease. + * + * However, we do not cut our short-term estimates lower than the current rate + * and volume of delivered data from this round trip, since from the current + * delivery process we can estimate the measured capacity available now. + * + * Anything faster than that approach would knowingly risk high loss, which can + * cause low bw for Reno/CUBIC and high loss recovery latency for + * request/response flows using any congestion control. + */ +static void bbr_adapt_lower_bounds(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_inflight_lo = ~0U; + + /* We only use lower-bound estimates when not probing bw. + * When probing we need to push inflight higher to probe bw. + */ + if (bbr_is_probing_bandwidth(sk)) + return; + + /* ECN response. */ + if (bbr->ecn_in_round && !!bbr_param(sk, ecn_factor)) { + bbr_init_lower_bounds(sk, false); + bbr_ecn_lower_bounds(sk, &ecn_inflight_lo); + } + + /* Loss response. */ + if (bbr->loss_in_round) { + bbr_init_lower_bounds(sk, true); + bbr_loss_lower_bounds(sk, &bbr->bw_lo, &bbr->inflight_lo); + } + + /* Adjust to the lower of the levels implied by loss/ECN. */ + bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); + bbr->bw_lo = max(1U, bbr->bw_lo); +} + +/* Reset any short-term lower-bound adaptation to congestion, so that we can + * push our inflight up. + */ +static void bbr_reset_lower_bounds(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_lo = ~0U; + bbr->inflight_lo = ~0U; +} + +/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state + * machine phase where we adapt our lower bound based on congestion signals. + */ +static void bbr_reset_congestion_signals(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->loss_in_cycle = 0; + bbr->ecn_in_cycle = 0; + bbr->bw_latest = 0; + bbr->inflight_latest = 0; +} + +static void bbr_exit_loss_recovery(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); + bbr->try_fast_path = 0; /* bound cwnd using latest model */ +} + +/* Update rate and volume of delivered data from latest round trip. */ +static void bbr_update_latest_delivery_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_round_start = 0; + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + + bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); + bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); + + if (!before(rs->prior_delivered, bbr->loss_round_delivered)) { + bbr->loss_round_delivered = tp->delivered; + bbr->loss_round_start = 1; /* mark start of new round trip */ + } +} + +/* Once per round, reset filter for latest rate and volume of delivered data. */ +static void bbr_advance_latest_delivery_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* If ACK matches a TLP retransmit, persist the filter. If we detect + * that a TLP retransmit plugged a tail loss, we'll want to remember + * how much data the path delivered before the tail loss. + */ + if (bbr->loss_round_start && !rs->is_acking_tlp_retrans_seq) { + bbr->bw_latest = ctx->sample_bw; + bbr->inflight_latest = rs->delivered; + } +} + +/* Update (most of) our congestion signals: track the recent rate and volume of + * delivered data, presence of loss, and EWMA degree of ECN marking. + */ +static void bbr_update_congestion_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + bw = ctx->sample_bw; + + if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) + bbr_take_max_bw_sample(sk, bw); + + bbr->loss_in_round |= (rs->losses > 0); + + if (!bbr->loss_round_start) + return; /* skip the per-round-trip updates */ + /* Now do per-round-trip updates. */ + bbr_adapt_lower_bounds(sk, rs); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; +} + +/* Bandwidth probing can cause loss. To help coexistence with loss-based + * congestion control we spread out our probing in a Reno-conscious way. Due to + * the shape of the Reno sawtooth, the time required between loss epochs for an + * idealized Reno flow is a number of round trips that is the BDP of that + * flow. We count packet-timed round trips directly, since measured RTT can + * vary widely, and Reno is driven by packet-timed round trips. + */ +static bool bbr_is_reno_coexistence_probe_time(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 rounds; + + /* Random loss can shave some small percentage off of our inflight + * in each round. To survive this, flows need robust periodic probes. + */ + rounds = min_t(u32, bbr_param(sk, bw_probe_max_rounds), bbr_target_inflight(sk)); + return bbr->rounds_since_probe >= rounds; +} + +/* How long do we want to wait before probing for bandwidth (and risking + * loss)? We randomize the wait, for better mixing and fairness convergence. + * + * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. + * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, + * (eg 4K video to a broadband user): + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + * + * We bound the BBR-native inter-bw-probe wall clock time to be: + * (a) higher than 2 sec: to try to avoid causing loss for a long enough time + * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must + * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs + * (b) lower than 3 sec: to ensure flows can start probing in a reasonable + * amount of time to discover unutilized bw on human-scale interactive + * time-scales (e.g. perhaps traffic from a web page download that we + * were competing with is now complete). + */ +static void bbr_pick_probe_wait(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Decide the random round-trip bound for wait until probe: */ + bbr->rounds_since_probe = + get_random_u32_below(bbr_param(sk, bw_probe_rand_rounds)); + /* Decide the random wall clock bound for wait until probe: */ + bbr->probe_wait_us = bbr_param(sk, bw_probe_base_us) + + get_random_u32_below(bbr_param(sk, bw_probe_rand_us)); +} + +static void bbr_set_cycle_idx(struct sock *sk, int cycle_idx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->cycle_idx = cycle_idx; + /* New phase, so need to update cwnd and pacing rate. */ + bbr->try_fast_path = 0; +} + +/* Send at estimated bw to fill the pipe, but not queue. We need this phase + * before PROBE_UP, because as soon as we send faster than the available bw + * we will start building a queue, and if the buffer is shallow we can cause + * loss. If we do not fill the pipe before we cause this loss, our bw_hi and + * inflight_hi estimates will underestimate. + */ +static void bbr_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_lower_bounds(sk); + bbr->bw_probe_up_rounds = bw_probe_up_rounds; + bbr->bw_probe_up_acks = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_REFILLING; + bbr->next_rtt_delivered = tp->delivered; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); +} + +/* Now probe max deliverable data rate and volume. */ +static void bbr_start_bw_probe_up(struct sock *sk, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->ack_phase = BBR_ACKS_PROBE_STARTING; + bbr->next_rtt_delivered = tp->delivered; + bbr->cycle_mstamp = tp->tcp_mstamp; + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_UP); + bbr_raise_inflight_hi_slope(sk); +} + +/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall + * clock time at which to probe beyond an inflight that we think to be + * safe. This will knowingly risk packet loss, so we want to do this rarely, to + * keep packet loss rates low. Also start a round-trip counter, to probe faster + * if we estimate a Reno flow at our BDP would probe faster. + */ +static void bbr_start_bw_probe_down(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_congestion_signals(sk); + bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ + bbr_pick_probe_wait(sk); + bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); +} + +/* Cruise: maintain what we estimate to be a neutral, conservative + * operating point, without attempting to probe up for bandwidth or down for + * RTT, and only reducing inflight in response to loss/ECN signals. + */ +static void bbr_start_bw_probe_cruise(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->inflight_lo != ~0U) + bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); + + bbr_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); +} + +/* Loss and/or ECN rate is too high while probing. + * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. + */ +static void bbr_handle_inflight_too_high(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + const u32 beta = bbr_param(sk, beta); + + bbr->prev_probe_too_high = 1; + bbr->bw_probe_samples = 0; /* only react once per probe */ + /* If we are app-limited then we are not robustly + * probing the max volume of inflight data we think + * might be safe (analogous to how app-limited bw + * samples are not known to be robustly probing bw). + */ + if (!rs->is_app_limited) { + bbr->inflight_hi = max_t(u32, rs->tx_in_flight, + (u64)bbr_target_inflight(sk) * + (BBR_UNIT - beta) >> BBR_SCALE); + } + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr_start_bw_probe_down(sk); +} + +/* If we're seeing bw and loss samples reflecting our bw probing, adapt + * using the signals we see. If loss or ECN mark rate gets too high, then adapt + * inflight_hi downward. If we're able to push inflight higher without such + * signals, push higher: adapt inflight_hi upward. + */ +static bool bbr_adapt_upper_bounds(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Track when we'll see bw/loss samples resulting from our bw probes. */ + if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) + bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; + if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { + /* End of samples from bw probing phase. */ + bbr->bw_probe_samples = 0; + bbr->ack_phase = BBR_ACKS_INIT; + /* At this point in the cycle, our current bw sample is also + * our best recent chance at finding the highest available bw + * for this flow. So now is the best time to forget the bw + * samples from the previous cycle, by advancing the window. + */ + if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) + bbr_advance_max_bw_filter(sk); + /* If we had an inflight_hi, then probed and pushed inflight all + * the way up to hit that inflight_hi without seeing any + * high loss/ECN in all the resulting ACKs from that probing, + * then probe up again, this time letting inflight persist at + * inflight_hi for a round trip, then accelerating beyond. + */ + if (bbr->mode == BBR_PROBE_BW && + bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { + bbr_start_bw_probe_refill(sk, 0); + return true; /* yes, decided state transition */ + } + } + if (bbr_is_inflight_too_high(sk, rs)) { + if (bbr->bw_probe_samples) /* sample is from bw probing? */ + bbr_handle_inflight_too_high(sk, rs); + } else { + /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ + + if (bbr->inflight_hi == ~0U) + return false; /* no excess queue signals yet */ + + /* To be resilient to random loss, we must raise bw/inflight_hi + * if we observe in any phase that a higher level is safe. + */ + if (rs->tx_in_flight > bbr->inflight_hi) { + bbr->inflight_hi = rs->tx_in_flight; + } + + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr_probe_inflight_hi_upward(sk, rs); + } + + return false; +} + +/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ +static bool bbr_check_time_to_probe_bw(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 n; + + /* If we seem to be at an operating point where we are not seeing loss + * but we are seeing ECN marks, then when the ECN marks cease we reprobe + * quickly (in case cross-traffic has ceased and freed up bw). + */ + if (bbr_param(sk, ecn_reprobe_gain) && bbr->ecn_eligible && + bbr->ecn_in_cycle && !bbr->loss_in_cycle && + inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { + /* Calculate n so that when bbr_raise_inflight_hi_slope() + * computes growth_this_round as 2^n it will be roughly the + * desired volume of data (inflight_hi*ecn_reprobe_gain). + */ + n = ilog2((((u64)bbr->inflight_hi * + bbr_param(sk, ecn_reprobe_gain)) >> BBR_SCALE)); + bbr_start_bw_probe_refill(sk, n); + return true; + } + + if (bbr_has_elapsed_in_phase(sk, bbr->probe_wait_us) || + bbr_is_reno_coexistence_probe_time(sk)) { + bbr_start_bw_probe_refill(sk, 0); + return true; + } + return false; +} + +/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ +static bool bbr_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) +{ + /* Always need to pull inflight down to leave headroom in queue. */ + if (inflight > bbr_inflight_with_headroom(sk)) + return false; + + return inflight <= bbr_inflight(sk, bw, BBR_UNIT); +} + +/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ +static void bbr_update_cycle_phase(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) { + struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); + bool is_bw_probe_done = false; + u32 inflight, bw; + + if (!bbr_full_bw_reached(sk)) + return; + + /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ + if (bbr_adapt_upper_bounds(sk, rs, ctx)) + return; /* already decided state transition */ + + if (bbr->mode != BBR_PROBE_BW) + return; + + inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); + bw = bbr_max_bw(sk); + + switch (bbr->cycle_idx) { + /* First we spend most of our time cruising with a pacing_gain of 1.0, + * which paces at the estimated bw, to try to fully use the pipe + * without building queue. If we encounter loss/ECN marks, we adapt + * by slowing down. + */ + case BBR_BW_PROBE_CRUISE: + if (bbr_check_time_to_probe_bw(sk, rs)) + return; /* already decided state transition */ + break; + + /* After cruising, when it's time to probe, we first "refill": we send + * at the estimated bw to fill the pipe, before probing higher and + * knowingly risking overflowing the bottleneck buffer (causing loss). + */ + case BBR_BW_PROBE_REFILL: + if (bbr->round_start) { + /* After one full round trip of sending in REFILL, we + * start to see bw samples reflecting our REFILL, which + * may be putting too much data in flight. + */ + bbr->bw_probe_samples = 1; + bbr_start_bw_probe_up(sk, ctx); + } + break; - bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ + /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to + * probe for bw. If we have not seen loss/ECN, we try to raise inflight + * to at least pacing_gain*BDP; note that this may take more than + * min_rtt if min_rtt is small (e.g. on a LAN). + * + * We terminate PROBE_UP bandwidth probing upon any of the following: + * + * (1) We've pushed inflight up to hit the inflight_hi target set in the + * most recent previous bw probe phase. Thus we want to start + * draining the queue immediately because it's very likely the most + * recently sent packets will fill the queue and cause drops. + * (2) If inflight_hi has not limited bandwidth growth recently, and + * yet delivered bandwidth has not increased much recently + * (bbr->full_bw_now). + * (3) Loss filter says loss rate is "too high". + * (4) ECN filter says ECN mark rate is "too high". + * + * (1) (2) checked here, (3) (4) checked in bbr_is_inflight_too_high() + */ + case BBR_BW_PROBE_UP: + if (bbr->prev_probe_too_high && + inflight >= bbr->inflight_hi) { + bbr->stopped_risky_probe = 1; + is_bw_probe_done = true; + } else { + if (tp->is_cwnd_limited && + tcp_snd_cwnd(tp) >= bbr->inflight_hi) { + /* inflight_hi is limiting bw growth */ + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + } else if (bbr->full_bw_now) { + /* Plateau in estimated bw. Pipe looks full. */ + is_bw_probe_done = true; + } + } + if (is_bw_probe_done) { + bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ + bbr_start_bw_probe_down(sk); /* restart w/ down */ + } + break; + + /* After probing in PROBE_UP, we have usually accumulated some data in + * the bottleneck buffer (if bw probing didn't find more bw). We next + * enter PROBE_DOWN to try to drain any excess data from the queue. To + * do this, we use a pacing_gain < 1.0. We hold this pacing gain until + * our inflight is less then that target cruising point, which is the + * minimum of (a) the amount needed to leave headroom, and (b) the + * estimated BDP. Once inflight falls to match the target, we estimate + * the queue is drained; persisting would underutilize the pipe. + */ + case BBR_BW_PROBE_DOWN: + if (bbr_check_time_to_probe_bw(sk, rs)) + return; /* already decided state transition */ + if (bbr_check_time_to_cruise(sk, inflight, bw)) + bbr_start_bw_probe_cruise(sk); + break; + + default: + WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); + } +} + +/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ +static void bbr_exit_probe_rtt(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_lower_bounds(sk); + if (bbr_full_bw_reached(sk)) { + bbr->mode = BBR_PROBE_BW; + /* Raising inflight after PROBE_RTT may cause loss, so reset + * the PROBE_BW clock and schedule the next bandwidth probe for + * a friendly and randomized future point in time. + */ + bbr_start_bw_probe_down(sk); + /* Since we are exiting PROBE_RTT, we know inflight is + * below our estimated BDP, so it is reasonable to cruise. + */ + bbr_start_bw_probe_cruise(sk); + } else { + bbr->mode = BBR_STARTUP; + } +} + +/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until + * the end of the round in recovery to get a good estimate of how many packets + * have been lost, and how many we need to drain with a low pacing rate. + */ +static void bbr_check_loss_too_high_in_startup(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk)) + return; + + /* For STARTUP exit, check the loss rate at the end of each round trip + * of Recovery episodes in STARTUP. We check the loss rate at the end + * of the round trip to filter out noisy/low loss and have a better + * sense of inflight (extent of loss), so we can drain more accurately. + */ + if (rs->losses && bbr->loss_events_in_round < 0xf) + bbr->loss_events_in_round++; /* update saturating counter */ + if (bbr_param(sk, full_loss_cnt) && bbr->loss_round_start && + inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && + bbr->loss_events_in_round >= bbr_param(sk, full_loss_cnt) && + bbr_is_inflight_too_high(sk, rs)) { + bbr_handle_queue_too_high_in_startup(sk); + return; + } + if (bbr->loss_round_start) + bbr->loss_events_in_round = 0; +} + +/* Estimate when the pipe is full, using the change in delivery rate: BBR + * estimates bw probing filled the pipe if the estimated bw hasn't changed by + * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited + * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the + * higher rwin, 3: we get higher delivery rate samples. Or transient + * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar + * design goal, but uses delay and inter-ACK spacing instead of bandwidth. + */ +static void bbr_check_full_bw_reached(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bw_thresh, full_cnt, thresh; + + if (bbr->full_bw_now || rs->is_app_limited) + return; + + thresh = bbr_param(sk, full_bw_thresh); + full_cnt = bbr_param(sk, full_bw_cnt); + bw_thresh = (u64)bbr->full_bw * thresh >> BBR_SCALE; + if (ctx->sample_bw >= bw_thresh) { + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + return; + } + if (!bbr->round_start) + return; + ++bbr->full_bw_cnt; + bbr->full_bw_now = bbr->full_bw_cnt >= full_cnt; + bbr->full_bw_reached |= bbr->full_bw_now; +} + +/* If pipe is probably full, drain the queue and then enter steady-state. */ +static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + /* Set ssthresh to export purely for monitoring, to signal + * completion of initial STARTUP by setting to a non- + * TCP_INFINITE_SSTHRESH value (ssthresh is not used by BBR). + */ + tcp_sk(sk)->snd_ssthresh = + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr_reset_congestion_signals(sk); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) { + bbr->mode = BBR_PROBE_BW; + bbr_start_bw_probe_down(sk); + } +} + +static void bbr_update_model(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + bbr_update_congestion_signals(sk, rs, ctx); + bbr_update_ack_aggregation(sk, rs); + bbr_check_loss_too_high_in_startup(sk, rs); + bbr_check_full_bw_reached(sk, rs, ctx); + bbr_check_drain(sk, rs, ctx); + bbr_update_cycle_phase(sk, rs, ctx); + bbr_update_min_rtt(sk, rs); +} + +/* Fast path for app-limited case. + * + * On each ack, we execute bbr state machine, which primarily consists of: + * 1) update model based on new rate sample, and + * 2) update control based on updated model or state change. + * + * There are certain workload/scenarios, e.g. app-limited case, where + * either we can skip updating model or we can skip update of both model + * as well as control. This provides signifcant softirq cpu savings for + * processing incoming acks. + * + * In case of app-limited, if there is no congestion (loss/ecn) and + * if observed bw sample is less than current estimated bw, then we can + * skip some of the computation in bbr state processing: + * + * - if there is no rtt/mode/phase change: In this case, since all the + * parameters of the network model are constant, we can skip model + * as well control update. + * + * - else we can skip rest of the model update. But we still need to + * update the control to account for the new rtt/mode/phase. + * + * Returns whether we can take fast path or not. + */ +static bool bbr_run_fast_path(struct sock *sk, bool *update_model, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 prev_min_rtt_us, prev_mode; + + if (bbr_param(sk, fast_path) && bbr->try_fast_path && + rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && + !bbr->loss_in_round && !bbr->ecn_in_round ) { + prev_mode = bbr->mode; + prev_min_rtt_us = bbr->min_rtt_us; + bbr_check_drain(sk, rs, ctx); + bbr_update_cycle_phase(sk, rs, ctx); + bbr_update_min_rtt(sk, rs); + + if (bbr->mode == prev_mode && + bbr->min_rtt_us == prev_min_rtt_us && + bbr->try_fast_path) { + return true; + } + + /* Skip model update, but control still needs to be updated */ + *update_model = false; + } + return false; +} + +__bpf_kfunc static void bbr_main(struct sock *sk, u32 ack, int flag, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct bbr_context ctx = { 0 }; + bool update_model = true; + u32 bw, round_delivered; + int ce_ratio = -1; + + round_delivered = bbr_update_round_start(sk, rs, &ctx); + if (bbr->round_start) { + bbr->rounds_since_probe = + min_t(s32, bbr->rounds_since_probe + 1, 0xFF); + ce_ratio = bbr_update_ecn_alpha(sk); + } + bbr_plb(sk, rs, ce_ratio); + + bbr->ecn_in_round |= (bbr->ecn_eligible && rs->is_ece); + bbr_calculate_bw_sample(sk, rs, &ctx); + bbr_update_latest_delivery_signals(sk, rs, &ctx); + + if (bbr_run_fast_path(sk, &update_model, rs, &ctx)) + goto out; + + if (update_model) + bbr_update_model(sk, rs, &ctx); + + bbr_update_gains(sk); + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, + tcp_snd_cwnd(tp), &ctx); + bbr_bound_cwnd_for_inflight_model(sk); + +out: + bbr_advance_latest_delivery_signals(sk, rs, &ctx); + bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; + bbr->loss_in_cycle |= rs->lost > 0; + bbr->ecn_in_cycle |= rs->delivered_ce > 0; +} + +__bpf_kfunc static void bbr_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->initialized = 1; + + bbr->init_cwnd = min(0x7FU, tcp_snd_cwnd(tp)); + bbr->prior_cwnd = tp->prior_cwnd; + tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + bbr->next_rtt_delivered = tp->delivered; + bbr->prev_ca_state = TCP_CA_Open; + + bbr->probe_rtt_done_stamp = 0; + bbr->probe_rtt_round_done = 0; + bbr->probe_rtt_min_us = tcp_min_rtt(tp); + bbr->probe_rtt_min_stamp = tcp_jiffies32; + bbr->min_rtt_us = tcp_min_rtt(tp); + bbr->min_rtt_stamp = tcp_jiffies32; + + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; + bbr->full_bw = 0; bbr->full_bw_cnt = 0; - bbr_reset_lt_bw_sampling(sk); - return tcp_snd_cwnd(tcp_sk(sk)); + bbr->cycle_mstamp = 0; + bbr->cycle_idx = 0; + + bbr_reset_startup_mode(sk); + + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = 0; + bbr->extra_acked[0] = 0; + bbr->extra_acked[1] = 0; + + bbr->ce_state = 0; + bbr->prior_rcv_nxt = tp->rcv_nxt; + bbr->try_fast_path = 0; + + cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + + /* Start sampling ECN mark rate after first full flight is ACKed: */ + bbr->loss_round_delivered = tp->delivered + 1; + bbr->loss_round_start = 0; + bbr->undo_bw_lo = 0; + bbr->undo_inflight_lo = 0; + bbr->undo_inflight_hi = 0; + bbr->loss_events_in_round = 0; + bbr->startup_ecn_rounds = 0; + bbr_reset_congestion_signals(sk); + bbr->bw_lo = ~0U; + bbr->bw_hi[0] = 0; + bbr->bw_hi[1] = 0; + bbr->inflight_lo = ~0U; + bbr->inflight_hi = ~0U; + bbr_reset_full_bw(sk); + bbr->bw_probe_up_cnt = ~0U; + bbr->bw_probe_up_acks = 0; + bbr->bw_probe_up_rounds = 0; + bbr->probe_wait_us = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_INIT; + bbr->rounds_since_probe = 0; + bbr->bw_probe_samples = 0; + bbr->prev_probe_too_high = 0; + bbr->ecn_eligible = 0; + bbr->ecn_alpha = bbr_param(sk, ecn_alpha_init); + bbr->alpha_last_delivered = 0; + bbr->alpha_last_delivered_ce = 0; + bbr->plb.pause_until = 0; + + tp->fast_ack_mode = bbr_fast_ack_mode ? 1 : 0; + + if (bbr_can_use_ecn(sk)) + tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; +} + +/* BBR marks the current round trip as a loss round. */ +static void bbr_note_loss(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + /* Capture "current" data over the full round trip of loss, to + * have a better chance of observing the full capacity of the path. + */ + if (!bbr->loss_in_round) /* first loss in this round trip? */ + bbr->loss_round_delivered = tp->delivered; /* set round trip */ + bbr->loss_in_round = 1; + bbr->loss_in_cycle = 1; } -/* Entering loss recovery, so save cwnd for when we exit or undo recovery. */ +/* Core TCP stack informs us that the given skb was just marked lost. */ +__bpf_kfunc static void bbr_skb_marked_lost(struct sock *sk, + const struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_skb_cb *scb = TCP_SKB_CB(skb); + struct rate_sample rs = {}; + + bbr_note_loss(sk); + + if (!bbr->bw_probe_samples) + return; /* not an skb sent while probing for bandwidth */ + if (unlikely(!scb->tx.delivered_mstamp)) + return; /* skb was SACKed, reneged, marked lost; ignore it */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this lost skb, + * then see if the loss rate went too high, and if so at which packet. + */ + rs.tx_in_flight = scb->tx.in_flight; + rs.lost = tp->lost - scb->tx.lost; + rs.is_app_limited = scb->tx.is_app_limited; + if (bbr_is_inflight_too_high(sk, &rs)) { + rs.tx_in_flight = bbr_inflight_hi_from_lost_skb(sk, &rs, skb); + bbr_handle_inflight_too_high(sk, &rs); + } +} + +static void bbr_run_loss_probe_recovery(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct rate_sample rs = {0}; + + bbr_note_loss(sk); + + if (!bbr->bw_probe_samples) + return; /* not sent while probing for bandwidth */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this + * loss, then see if the loss rate went too high. + */ + rs.lost = 1; /* TLP probe repaired loss of a single segment */ + rs.tx_in_flight = bbr->inflight_latest + rs.lost; + rs.is_app_limited = tp->tlp_orig_data_app_limited; + if (bbr_is_inflight_too_high(sk, &rs)) + bbr_handle_inflight_too_high(sk, &rs); +} + +/* Revert short-term model if current loss recovery event was spurious. */ +__bpf_kfunc static u32 bbr_undo_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_full_bw(sk); /* spurious slow-down; reset full bw detector */ + bbr->loss_in_round = 0; + + /* Revert to cwnd and other state saved before loss episode. */ + bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); + bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); + bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); + bbr->try_fast_path = 0; /* take slow path to set proper cwnd, pacing */ + return bbr->prior_cwnd; +} + +/* Entering loss recovery, so save state for when we undo recovery. */ __bpf_kfunc static u32 bbr_ssthresh(struct sock *sk) { + struct bbr *bbr = inet_csk_ca(sk); + bbr_save_cwnd(sk); + /* For undo, save state that adapts based on loss signal. */ + bbr->undo_bw_lo = bbr->bw_lo; + bbr->undo_inflight_lo = bbr->inflight_lo; + bbr->undo_inflight_hi = bbr->inflight_hi; return tcp_sk(sk)->snd_ssthresh; } +static enum tcp_bbr_phase bbr_get_phase(struct bbr *bbr) +{ + switch (bbr->mode) { + case BBR_STARTUP: + return BBR_PHASE_STARTUP; + case BBR_DRAIN: + return BBR_PHASE_DRAIN; + case BBR_PROBE_BW: + break; + case BBR_PROBE_RTT: + return BBR_PHASE_PROBE_RTT; + default: + return BBR_PHASE_INVALID; + } + switch (bbr->cycle_idx) { + case BBR_BW_PROBE_UP: + return BBR_PHASE_PROBE_BW_UP; + case BBR_BW_PROBE_DOWN: + return BBR_PHASE_PROBE_BW_DOWN; + case BBR_BW_PROBE_CRUISE: + return BBR_PHASE_PROBE_BW_CRUISE; + case BBR_BW_PROBE_REFILL: + return BBR_PHASE_PROBE_BW_REFILL; + default: + return BBR_PHASE_INVALID; + } +} + static size_t bbr_get_info(struct sock *sk, u32 ext, int *attr, - union tcp_cc_info *info) + union tcp_cc_info *info) { if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || ext & (1 << (INET_DIAG_VEGASINFO - 1))) { - struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u64 bw = bbr_bw(sk); - - bw = bw * tp->mss_cache * USEC_PER_SEC >> BW_SCALE; - memset(&info->bbr, 0, sizeof(info->bbr)); - info->bbr.bbr_bw_lo = (u32)bw; - info->bbr.bbr_bw_hi = (u32)(bw >> 32); - info->bbr.bbr_min_rtt = bbr->min_rtt_us; - info->bbr.bbr_pacing_gain = bbr->pacing_gain; - info->bbr.bbr_cwnd_gain = bbr->cwnd_gain; + u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); + u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); + u64 bw_lo = bbr->bw_lo == ~0U ? + ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); + struct tcp_bbr_info *bbr_info = &info->bbr; + + memset(bbr_info, 0, sizeof(*bbr_info)); + bbr_info->bbr_bw_lo = (u32)bw; + bbr_info->bbr_bw_hi = (u32)(bw >> 32); + bbr_info->bbr_min_rtt = bbr->min_rtt_us; + bbr_info->bbr_pacing_gain = bbr->pacing_gain; + bbr_info->bbr_cwnd_gain = bbr->cwnd_gain; + bbr_info->bbr_bw_hi_lsb = (u32)bw_hi; + bbr_info->bbr_bw_hi_msb = (u32)(bw_hi >> 32); + bbr_info->bbr_bw_lo_lsb = (u32)bw_lo; + bbr_info->bbr_bw_lo_msb = (u32)(bw_lo >> 32); + bbr_info->bbr_mode = bbr->mode; + bbr_info->bbr_phase = (__u8)bbr_get_phase(bbr); + bbr_info->bbr_version = (__u8)BBR_VERSION; + bbr_info->bbr_inflight_lo = bbr->inflight_lo; + bbr_info->bbr_inflight_hi = bbr->inflight_hi; + bbr_info->bbr_extra_acked = bbr_extra_acked(sk); *attr = INET_DIAG_BBRINFO; - return sizeof(info->bbr); + return sizeof(*bbr_info); } return 0; } __bpf_kfunc static void bbr_set_state(struct sock *sk, u8 new_state) { + struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); if (new_state == TCP_CA_Loss) { - struct rate_sample rs = { .losses = 1 }; bbr->prev_ca_state = TCP_CA_Loss; - bbr->full_bw = 0; - bbr->round_start = 1; /* treat RTO like end of a round */ - bbr_lt_bw_sampling(sk, &rs); + tcp_plb_update_state_upon_rto(sk, &bbr->plb); + /* The tcp_write_timeout() call to sk_rethink_txhash() likely + * repathed this flow, so re-learn the min network RTT on the + * new path: + */ + bbr_reset_full_bw(sk); + if (!bbr_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { + /* bbr_adapt_lower_bounds() needs cwnd before + * we suffered an RTO, to update inflight_lo: + */ + bbr->inflight_lo = + max(tcp_snd_cwnd(tp), bbr->prior_cwnd); + } + } else if (bbr->prev_ca_state == TCP_CA_Loss && + new_state != TCP_CA_Loss) { + bbr_exit_loss_recovery(sk); } } + static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { - .flags = TCP_CONG_NON_RESTRICTED, + .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, .name = "bbr", .owner = THIS_MODULE, .init = bbr_init, .cong_control = bbr_main, .sndbuf_expand = bbr_sndbuf_expand, + .skb_marked_lost = bbr_skb_marked_lost, .undo_cwnd = bbr_undo_cwnd, .cwnd_event = bbr_cwnd_event, .ssthresh = bbr_ssthresh, - .min_tso_segs = bbr_min_tso_segs, + .tso_segs = bbr_tso_segs, .get_info = bbr_get_info, .set_state = bbr_set_state, }; @@ -1159,10 +2361,11 @@ BTF_KFUNCS_START(tcp_bbr_check_kfunc_ids) BTF_ID_FLAGS(func, bbr_init) BTF_ID_FLAGS(func, bbr_main) BTF_ID_FLAGS(func, bbr_sndbuf_expand) +BTF_ID_FLAGS(func, bbr_skb_marked_lost) BTF_ID_FLAGS(func, bbr_undo_cwnd) BTF_ID_FLAGS(func, bbr_cwnd_event) BTF_ID_FLAGS(func, bbr_ssthresh) -BTF_ID_FLAGS(func, bbr_min_tso_segs) +BTF_ID_FLAGS(func, bbr_tso_segs) BTF_ID_FLAGS(func, bbr_set_state) BTF_KFUNCS_END(tcp_bbr_check_kfunc_ids) @@ -1195,5 +2398,12 @@ MODULE_AUTHOR("Van Jacobson "); MODULE_AUTHOR("Neal Cardwell "); MODULE_AUTHOR("Yuchung Cheng "); MODULE_AUTHOR("Soheil Hassas Yeganeh "); +MODULE_AUTHOR("Priyaranjan Jha "); +MODULE_AUTHOR("Yousuk Seung "); +MODULE_AUTHOR("Kevin Yang "); +MODULE_AUTHOR("Arjun Roy "); +MODULE_AUTHOR("David Morley "); + MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); +MODULE_VERSION(__stringify(BBR_VERSION)); diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index df758adbb445..e98e5dbc050e 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -237,6 +237,7 @@ void tcp_init_congestion_control(struct sock *sk) struct inet_connection_sock *icsk = inet_csk(sk); tcp_sk(sk)->prior_ssthresh = 0; + tcp_sk(sk)->fast_ack_mode = 0; if (icsk->icsk_ca_ops->init) icsk->icsk_ca_ops->init(sk); if (tcp_ca_needs_ecn(sk)) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 94391f32a5d8..6d9963172fbe 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -381,7 +381,7 @@ static void tcp_data_ecn_check(struct sock *sk, const struct sk_buff *skb) tcp_enter_quickack_mode(sk, 2); break; case INET_ECN_CE: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { @@ -392,7 +392,7 @@ static void tcp_data_ecn_check(struct sock *sk, const struct sk_buff *skb) tp->ecn_flags |= TCP_ECN_SEEN; break; default: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); tp->ecn_flags |= TCP_ECN_SEEN; break; @@ -1134,7 +1134,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) */ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) { + struct sock *sk = (struct sock *)tp; + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; + tp->lost += tcp_skb_pcount(skb); + if (ca_ops->skb_marked_lost) + ca_ops->skb_marked_lost(sk, skb); } void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) @@ -1506,6 +1511,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); tcp_skb_pcount_add(skb, -pcount); + /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ + if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, + "prev in_flight: %u skb in_flight: %u pcount: %u", + TCP_SKB_CB(prev)->tx.in_flight, + TCP_SKB_CB(skb)->tx.in_flight, + pcount)) + TCP_SKB_CB(skb)->tx.in_flight = 0; + else + TCP_SKB_CB(skb)->tx.in_flight -= pcount; + TCP_SKB_CB(prev)->tx.in_flight += pcount; + /* When we're adding to gso_segs == 1, gso_size will be zero, * in theory this shouldn't be necessary but as long as DSACK * code can come after this skb later on it's better to keep @@ -3856,7 +3872,8 @@ static int tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq) /* This routine deals with acks during a TLP episode and ends an episode by * resetting tlp_high_seq. Ref: TLP algorithm in draft-ietf-tcpm-rack */ -static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) +static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag, + struct rate_sample *rs) { struct tcp_sock *tp = tcp_sk(sk); @@ -3873,6 +3890,7 @@ static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) /* ACK advances: there was a loss, so reduce cwnd. Reset * tlp_high_seq in tcp_init_cwnd_reduction() */ + tcp_ca_event(sk, CA_EVENT_TLP_RECOVERY); tcp_init_cwnd_reduction(sk); tcp_set_ca_state(sk, TCP_CA_CWR); tcp_end_cwnd_reduction(sk); @@ -3883,6 +3901,11 @@ static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) FLAG_NOT_DUP | FLAG_DATA_SACKED))) { /* Pure dupack: original and TLP probe arrived; no loss */ tp->tlp_high_seq = 0; + } else { + /* This ACK matches a TLP retransmit. We cannot yet tell if + * this ACK is for the original or the TLP retransmit. + */ + rs->is_acking_tlp_retrans_seq = 1; } } @@ -4002,6 +4025,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; rs.prior_in_flight = tcp_packets_in_flight(tp); + tcp_rate_check_app_limited(sk); /* ts_recent update must be made after we are sure that the packet * is in window. @@ -4067,7 +4091,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) tcp_in_ack_event(sk, flag); if (tp->tlp_high_seq) - tcp_process_tlp_ack(sk, ack, flag); + tcp_process_tlp_ack(sk, ack, flag, &rs); if (tcp_ack_is_dubious(sk, flag)) { if (!(flag & (FLAG_SND_UNA_ADVANCED | @@ -4091,6 +4115,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) delivered = tcp_newly_delivered(sk, delivered, flag); lost = tp->lost - lost; /* freshly marked lost */ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + rs.is_ece = !!(flag & FLAG_ECE); tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); tcp_xmit_recovery(sk, rexmit); @@ -4111,7 +4136,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) tcp_ack_probe(sk); if (tp->tlp_high_seq) - tcp_process_tlp_ack(sk, ack, flag); + tcp_process_tlp_ack(sk, ack, flag, &rs); return 1; old_ack: @@ -5795,13 +5820,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) /* More than one full frame received... */ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && + (tp->fast_ack_mode == 1 || /* ... and right edge of window advances far enough. * (tcp_recvmsg() will send ACK otherwise). * If application uses SO_RCVLOWAT, we want send ack now if * we have not received enough bytes to satisfy the condition. */ - (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || - __tcp_select_window(sk) >= tp->rcv_wnd)) || + (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || + __tcp_select_window(sk) >= tp->rcv_wnd))) || /* We ACK each frame or... */ tcp_in_quickack_mode(sk) || /* Protocol state mandates a one-time immediate ACK */ diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index 43d7852ce07e..df386419b9bf 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -475,6 +475,8 @@ void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst) u32 ca_key = dst_metric(dst, RTAX_CC_ALGO); bool ca_got_dst = false; + tcp_set_ecn_low_from_dst(sk, dst); + if (ca_key != TCP_CA_UNSPEC) { const struct tcp_congestion_ops *ca; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 3ac8d2d17e1f..cc75963b5a4c 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -339,10 +339,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk); bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 || tcp_ca_needs_ecn(sk) || bpf_needs_ecn; + const struct dst_entry *dst = __sk_dst_get(sk); if (!use_ecn) { - const struct dst_entry *dst = __sk_dst_get(sk); - if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) use_ecn = true; } @@ -354,6 +353,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) INET_ECN_xmit(sk); + + if (dst) + tcp_set_ecn_low_from_dst(sk, dst); } } @@ -391,7 +393,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, th->cwr = 1; skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; } - } else if (!tcp_ca_needs_ecn(sk)) { + } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && + !tcp_ca_needs_ecn(sk)) { /* ACK or retransmitted segment: clear ECT|CE */ INET_ECN_dontxmit(sk); } @@ -1614,7 +1617,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, { struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *buff; - int old_factor; + int old_factor, inflight_prev; long limit; u16 flags; int nlen; @@ -1689,6 +1692,30 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, if (diff) tcp_adjust_pcount(sk, skb, diff); + + inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; + if (inflight_prev < 0) { + WARN_ONCE(tcp_skb_tx_in_flight_is_suspicious( + old_factor, + TCP_SKB_CB(skb)->sacked, + TCP_SKB_CB(skb)->tx.in_flight), + "inconsistent: tx.in_flight: %u " + "old_factor: %d mss: %u sacked: %u " + "1st pcount: %d 2nd pcount: %d " + "1st len: %u 2nd len: %u ", + TCP_SKB_CB(skb)->tx.in_flight, old_factor, + mss_now, TCP_SKB_CB(skb)->sacked, + tcp_skb_pcount(skb), tcp_skb_pcount(buff), + skb->len, buff->len); + inflight_prev = 0; + } + /* Set 1st tx.in_flight as if 1st were sent by itself: */ + TCP_SKB_CB(skb)->tx.in_flight = inflight_prev + + tcp_skb_pcount(skb); + /* Set 2nd tx.in_flight with new 1st and 2nd pcounts: */ + TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + + tcp_skb_pcount(skb) + + tcp_skb_pcount(buff); } /* Link BUFF into the send queue. */ @@ -2045,13 +2072,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) { const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; - u32 min_tso, tso_segs; - - min_tso = ca_ops->min_tso_segs ? - ca_ops->min_tso_segs(sk) : - READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + u32 tso_segs; - tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + tso_segs = ca_ops->tso_segs ? + ca_ops->tso_segs(sk, mss_now) : + tcp_tso_autosize(sk, mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); return min_t(u32, tso_segs, sk->sk_gso_max_segs); } @@ -2776,6 +2802,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC); list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); tcp_init_tso_segs(skb, mss_now); + tcp_set_tx_in_flight(sk, skb); goto repair; /* Skip network transmission */ } @@ -2988,6 +3015,7 @@ void tcp_send_loss_probe(struct sock *sk) if (WARN_ON(!skb || !tcp_skb_pcount(skb))) goto rearm_timer; + tp->tlp_orig_data_app_limited = TCP_SKB_CB(skb)->tx.is_app_limited; if (__tcp_retransmit_skb(sk, skb, 1)) goto rearm_timer; diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c index a8f6d9d06f2e..8737f2134648 100644 --- a/net/ipv4/tcp_rate.c +++ b/net/ipv4/tcp_rate.c @@ -34,6 +34,24 @@ * ready to send in the write queue. */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + u32 in_flight; + + /* Check, sanitize, and record packets in flight after skb was sent. */ + in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); + if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, + "insane in_flight %u cc %s mss %u " + "cwnd %u pif %u %u %u %u\n", + in_flight, inet_csk(sk)->icsk_ca_ops->name, + tp->mss_cache, tp->snd_cwnd, + tp->packets_out, tp->retrans_out, + tp->sacked_out, tp->lost_out)) + in_flight = TCPCB_IN_FLIGHT_MAX; + TCP_SKB_CB(skb)->tx.in_flight = in_flight; +} + /* Snapshot the current delivery information in the skb, to generate * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). */ @@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; TCP_SKB_CB(skb)->tx.delivered = tp->delivered; TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; + TCP_SKB_CB(skb)->tx.lost = tp->lost; TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; + tcp_set_tx_in_flight(sk, skb); } /* When an skb is sacked or acked, we fill in the rate sample with the (prior) @@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, if (!rs->prior_delivered || tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, scb->end_seq, rs->last_end_seq)) { + rs->prior_lost = scb->tx.lost; rs->prior_delivered_ce = scb->tx.delivered_ce; rs->prior_delivered = scb->tx.delivered; rs->prior_mstamp = scb->tx.delivered_mstamp; rs->is_app_limited = scb->tx.is_app_limited; rs->is_retrans = scb->sacked & TCPCB_RETRANS; + rs->tx_in_flight = scb->tx.in_flight; rs->last_end_seq = scb->end_seq; /* Record send time of most recently ACKed packet: */ tp->first_tx_mstamp = tx_tstamp; /* Find the duration of the "send phase" of this window: */ - rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, - scb->tx.first_tx_mstamp); + rs->interval_us = tcp_stamp32_us_delta( + tp->first_tx_mstamp, + scb->tx.first_tx_mstamp); } /* Mark off the skb delivered once it's sacked to avoid being @@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, return; } rs->delivered = tp->delivered - rs->prior_delivered; + rs->lost = tp->lost - rs->prior_lost; rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; /* delivered_ce occupies less than 32 bits in the skb control block */ @@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, * longer phase. */ snd_us = rs->interval_us; /* send phase */ - ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, + ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, rs->prior_mstamp); /* ack phase */ rs->interval_us = max(snd_us, ack_us); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index bb37e24b97a7..9adfc1131d1f 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -565,7 +565,7 @@ void tcp_retransmit_timer(struct sock *sk) struct inet_sock *inet = inet_sk(sk); u32 rtx_delta; - rtx_delta = tcp_time_stamp_ts(tp) - (tp->retrans_stamp ?: + rtx_delta = tcp_time_stamp_ts(tp) - (tp->retrans_stamp ?: tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb)); if (tp->tcp_usec_ts) rtx_delta /= USEC_PER_MSEC; @@ -702,6 +702,8 @@ void tcp_write_timer_handler(struct sock *sk) icsk_timeout(icsk)); return; } + + tcp_rate_check_app_limited(sk); tcp_mstamp_refresh(tcp_sk(sk)); event = icsk->icsk_pending; -- 2.51.0 From 498d8be89509c1d4cd481b35047403b61b4af683 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 18:19:35 +0200 Subject: [PATCH 3/6] block Signed-off-by: Peter Jung --- block/bfq-iosched.c | 52 +++++++++++++++++++++++++++++++++++++++------ block/bfq-iosched.h | 12 +++++++++-- block/mq-deadline.c | 48 +++++++++++++++++++++++++++++++++++------ 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 9483c529c958..6dd6a8f131a8 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -467,6 +467,21 @@ static struct bfq_io_cq *bfq_bic_lookup(struct request_queue *q) return icq; } +static struct bfq_io_cq *bfq_bic_try_lookup(struct request_queue *q) +{ + if (!current->io_context) + return NULL; + if (spin_trylock_irq(&q->queue_lock)) { + struct bfq_io_cq *icq; + + icq = icq_to_bic(ioc_lookup_icq(q)); + spin_unlock_irq(&q->queue_lock); + return icq; + } + + return NULL; +} + /* * Scheduler run of queue, if there are requests pending and no one in the * driver that will restart queueing. @@ -2463,10 +2478,21 @@ static bool bfq_bio_merge(struct request_queue *q, struct bio *bio, * returned by bfq_bic_lookup does not go away before * bfqd->lock is taken. */ - struct bfq_io_cq *bic = bfq_bic_lookup(q); + struct bfq_io_cq *bic = bfq_bic_try_lookup(q); bool ret; - spin_lock_irq(&bfqd->lock); + /* + * bio merging is called for every bio queued, and it's very easy + * to run into contention because of that. If we fail getting + * the dd lock, just skip this merge attempt. For related IO, the + * plug will be the successful merging point. If we get here, we + * already failed doing the obvious merge. Chances of actually + * getting a merge off this path is a lot slimmer, so skipping an + * occassional lookup that will most likely not succeed anyway should + * not be a problem. + */ + if (!spin_trylock_irq(&bfqd->lock)) + return false; if (bic) { /* @@ -5315,6 +5341,18 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) struct bfq_queue *in_serv_queue; bool waiting_rq, idle_timer_disabled = false; + /* + * If someone else is already dispatching, skip this one. This will + * defer the next dispatch event to when something completes, and could + * potentially lower the queue depth for contended cases. + * + * See the logic in blk_mq_do_dispatch_sched(), which loops and + * retries if nothing is dispatched. + */ + if (test_bit(BFQ_DISPATCHING, &bfqd->run_state) || + test_and_set_bit_lock(BFQ_DISPATCHING, &bfqd->run_state)) + return NULL; + spin_lock_irq(&bfqd->lock); in_serv_queue = bfqd->in_service_queue; @@ -5326,6 +5364,7 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) waiting_rq && !bfq_bfqq_wait_request(in_serv_queue); } + clear_bit_unlock(BFQ_DISPATCHING, &bfqd->run_state); spin_unlock_irq(&bfqd->lock); bfq_update_dispatch_stats(hctx->queue, rq, idle_timer_disabled ? in_serv_queue : NULL, @@ -6248,10 +6287,9 @@ static inline void bfq_update_insert_stats(struct request_queue *q, static struct bfq_queue *bfq_init_rq(struct request *rq); -static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, +static void bfq_insert_request(struct request_queue *q, struct request *rq, blk_insert_t flags) { - struct request_queue *q = hctx->queue; struct bfq_data *bfqd = q->elevator->elevator_data; struct bfq_queue *bfqq; bool idle_timer_disabled = false; @@ -6313,7 +6351,7 @@ static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); - bfq_insert_request(hctx, rq, flags); + bfq_insert_request(hctx->queue, rq, flags); } } @@ -7245,6 +7283,8 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) q->elevator = eq; spin_unlock_irq(&q->queue_lock); + spin_lock_init(&bfqd->lock); + /* * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. * Grab a permanent reference to it, so that the normal code flow @@ -7362,8 +7402,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) /* see comments on the definition of next field inside bfq_data */ bfqd->actuator_load_threshold = 4; - spin_lock_init(&bfqd->lock); - /* * The invocation of the next bfq_create_group_hierarchy * function is the head of a chain of function calls diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index 31217f196f4f..a250c559976f 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -504,12 +504,22 @@ struct bfq_io_cq { unsigned int requests; /* Number of requests this process has in flight */ }; +enum { + BFQ_DISPATCHING = 0, +}; + /** * struct bfq_data - per-device data structure. * * All the fields are protected by @lock. */ struct bfq_data { + struct { + spinlock_t lock; + } ____cacheline_aligned_in_smp; + + unsigned long run_state; + /* device request queue */ struct request_queue *queue; /* dispatch queue */ @@ -795,8 +805,6 @@ struct bfq_data { /* fallback dummy bfqq for extreme OOM conditions */ struct bfq_queue oom_bfqq; - spinlock_t lock; - /* * bic associated with the task issuing current bio for * merging. This and the next field are used as a support to diff --git a/block/mq-deadline.c b/block/mq-deadline.c index b9b7cdf1d3c9..91e5acde679a 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -79,10 +79,20 @@ struct dd_per_prio { struct io_stats_per_prio stats; }; +enum { + DD_DISPATCHING = 0, +}; + struct deadline_data { /* * run time data */ + struct { + spinlock_t lock; + spinlock_t zone_lock; + } ____cacheline_aligned_in_smp; + + unsigned long run_state; struct dd_per_prio per_prio[DD_PRIO_COUNT]; @@ -100,8 +110,6 @@ struct deadline_data { int front_merges; u32 async_depth; int prio_aging_expire; - - spinlock_t lock; }; /* Maps an I/O priority class to a deadline scheduler priority. */ @@ -466,6 +474,18 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) struct request *rq; enum dd_prio prio; + /* + * If someone else is already dispatching, skip this one. This will + * defer the next dispatch event to when something completes, and could + * potentially lower the queue depth for contended cases. + * + * See the logic in blk_mq_do_dispatch_sched(), which loops and + * retries if nothing is dispatched. + */ + if (test_bit(DD_DISPATCHING, &dd->run_state) || + test_and_set_bit_lock(DD_DISPATCHING, &dd->run_state)) + return NULL; + spin_lock(&dd->lock); rq = dd_dispatch_prio_aged_requests(dd, now); if (rq) @@ -482,6 +502,7 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) } unlock: + clear_bit_unlock(DD_DISPATCHING, &dd->run_state); spin_unlock(&dd->lock); return rq; @@ -565,6 +586,9 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) eq->elevator_data = dd; + spin_lock_init(&dd->lock); + spin_lock_init(&dd->zone_lock); + for (prio = 0; prio <= DD_PRIO_MAX; prio++) { struct dd_per_prio *per_prio = &dd->per_prio[prio]; @@ -581,7 +605,6 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) dd->last_dir = DD_WRITE; dd->fifo_batch = fifo_batch; dd->prio_aging_expire = prio_aging_expire; - spin_lock_init(&dd->lock); /* We dispatch from request queue wide instead of hw queue */ blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); @@ -633,7 +656,19 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio, struct request *free = NULL; bool ret; - spin_lock(&dd->lock); + /* + * bio merging is called for every bio queued, and it's very easy + * to run into contention because of that. If we fail getting + * the dd lock, just skip this merge attempt. For related IO, the + * plug will be the successful merging point. If we get here, we + * already failed doing the obvious merge. Chances of actually + * getting a merge off this path is a lot slimmer, so skipping an + * occassional lookup that will most likely not succeed anyway should + * not be a problem. + */ + if (!spin_trylock(&dd->lock)) + return false; + ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); spin_unlock(&dd->lock); @@ -646,10 +681,9 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio, /* * add rq to rbtree and fifo */ -static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, +static void dd_insert_request(struct request_queue *q, struct request *rq, blk_insert_t flags, struct list_head *free) { - struct request_queue *q = hctx->queue; struct deadline_data *dd = q->elevator->elevator_data; const enum dd_data_dir data_dir = rq_data_dir(rq); u16 ioprio = req_get_ioprio(rq); @@ -707,7 +741,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); - dd_insert_request(hctx, rq, flags, &free); + dd_insert_request(q, rq, flags, &free); } spin_unlock(&dd->lock); -- 2.51.0 From db3822cf9b94d72023fc485c20b5f5e3b0f2473e Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 18:19:48 +0200 Subject: [PATCH 4/6] cachy Signed-off-by: Peter Jung --- .gitignore | 2 + .../admin-guide/kernel-parameters.txt | 12 + Documentation/admin-guide/sysctl/vm.rst | 72 + Makefile | 33 +- arch/Kconfig | 19 + arch/x86/Kconfig.cpu | 46 + arch/x86/Makefile | 16 +- arch/x86/include/asm/pci.h | 6 + arch/x86/pci/common.c | 7 +- block/Kconfig.iosched | 14 + block/Makefile | 8 + block/adios.c | 1881 ++++++++++ block/elevator.c | 26 +- drivers/Makefile | 13 +- drivers/ata/ahci.c | 23 +- drivers/cpufreq/Kconfig.x86 | 2 - drivers/cpufreq/intel_pstate.c | 2 + drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 10 + drivers/gpu/drm/amd/display/Kconfig | 6 + .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 2 +- .../amd/display/amdgpu_dm/amdgpu_dm_color.c | 2 +- .../amd/display/amdgpu_dm/amdgpu_dm_crtc.c | 6 +- .../amd/display/amdgpu_dm/amdgpu_dm_plane.c | 6 +- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 3 + drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 14 +- drivers/hid/amd-sfh-hid/amd_sfh_client.c | 23 + drivers/hid/amd-sfh-hid/amd_sfh_hid.h | 2 +- drivers/hid/amd-sfh-hid/amd_sfh_pcie.c | 4 + drivers/hid/amd-sfh-hid/amd_sfh_pcie.h | 1 + drivers/input/evdev.c | 19 +- drivers/media/v4l2-core/Kconfig | 5 + drivers/media/v4l2-core/Makefile | 2 + drivers/media/v4l2-core/v4l2loopback.c | 3313 +++++++++++++++++ drivers/media/v4l2-core/v4l2loopback.h | 108 + .../media/v4l2-core/v4l2loopback_formats.h | 445 +++ drivers/pci/controller/Makefile | 6 + drivers/pci/controller/intel-nvme-remap.c | 462 +++ drivers/pci/quirks.c | 101 + drivers/scsi/Kconfig | 2 + drivers/scsi/Makefile | 1 + drivers/scsi/vhba/Kconfig | 9 + drivers/scsi/vhba/Makefile | 4 + drivers/scsi/vhba/vhba.c | 1132 ++++++ include/linux/mm.h | 10 +- include/linux/pagemap.h | 2 +- include/linux/user_namespace.h | 4 + include/linux/wait.h | 2 + init/Kconfig | 26 + kernel/Kconfig.hz | 24 + kernel/Kconfig.preempt | 2 +- kernel/fork.c | 14 + kernel/locking/rwsem.c | 4 +- kernel/sched/fair.c | 13 + kernel/sched/sched.h | 2 +- kernel/sched/wait.c | 24 + kernel/sysctl.c | 12 + kernel/user_namespace.c | 7 + mm/Kconfig | 65 +- mm/compaction.c | 4 + mm/huge_memory.c | 4 + mm/mm_init.c | 1 + mm/page-writeback.c | 8 + mm/page_alloc.c | 4 + mm/swap.c | 5 + mm/util.c | 34 + mm/vmpressure.c | 4 + mm/vmscan.c | 157 +- net/ipv4/inet_connection_sock.c | 2 +- scripts/Makefile.thinlto | 38 + scripts/Makefile.vmlinux_a | 83 + scripts/mod/modpost.c | 15 +- 72 files changed, 8362 insertions(+), 79 deletions(-) create mode 100644 block/adios.c create mode 100644 drivers/media/v4l2-core/v4l2loopback.c create mode 100644 drivers/media/v4l2-core/v4l2loopback.h create mode 100644 drivers/media/v4l2-core/v4l2loopback_formats.h create mode 100644 drivers/pci/controller/intel-nvme-remap.c create mode 100644 drivers/scsi/vhba/Kconfig create mode 100644 drivers/scsi/vhba/Makefile create mode 100644 drivers/scsi/vhba/vhba.c create mode 100644 scripts/Makefile.thinlto create mode 100644 scripts/Makefile.vmlinux_a diff --git a/.gitignore b/.gitignore index 929054df5212..e4b492cc3993 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ *.zst Module.symvers dtbs-list +builtin.order modules.order # @@ -66,6 +67,7 @@ modules.order /vmlinux.32 /vmlinux.map /vmlinux.symvers +/vmlinux.thinlto-index /vmlinux.unstripped /vmlinux-gdb.py /vmlinuz diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index f6d317e1674d..5856aa83a2ad 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2336,6 +2336,9 @@ disable Do not enable intel_pstate as the default scaling driver for the supported processors + enable + Enable intel_pstate in-case "disable" was passed + previously in the kernel boot parameters active Use intel_pstate driver to bypass the scaling governors layer of cpufreq and provides it own @@ -4732,6 +4735,15 @@ nomsi [MSI] If the PCI_MSI kernel config parameter is enabled, this kernel boot option can be used to disable the use of MSI interrupts system-wide. + pcie_acs_override = + [PCIE] Override missing PCIe ACS support for: + downstream + All downstream ports - full ACS capabilities + multfunction + All multifunction devices - multifunction ACS subset + id:nnnn:nnnn + Specfic device - full ACS capabilities + Specified as vid:did (vendor/device ID) in hex noioapicquirk [APIC] Disable all boot interrupt quirks. Safety option to keep boot IRQs enabled. This should never be necessary. diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 9bef46151d53..0ee649d4808b 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -25,6 +25,9 @@ files can be found in mm/swap.c. Currently, these files are in /proc/sys/vm: - admin_reserve_kbytes +- anon_min_ratio +- clean_low_ratio +- clean_min_ratio - compact_memory - compaction_proactiveness - compact_unevictable_allowed @@ -110,6 +113,67 @@ On x86_64 this is about 128MB. Changing this takes effect whenever an application requests memory. +anon_min_ratio +============== + +This knob provides *hard* protection of anonymous pages. The anonymous pages +on the current node won't be reclaimed under any conditions when their amount +is below vm.anon_min_ratio. + +This knob may be used to prevent excessive swap thrashing when anonymous +memory is low (for example, when memory is going to be overfilled by +compressed data of zram module). + +Setting this value too high (close to 100) can result in inability to +swap and can lead to early OOM under memory pressure. + +The unit of measurement is the percentage of the total memory of the node. + +The default value is 1. + + +clean_low_ratio +================ + +This knob provides *best-effort* protection of clean file pages. The file pages +on the current node won't be reclaimed under memory pressure when the amount of +clean file pages is below vm.clean_low_ratio *unless* we threaten to OOM. + +Protection of clean file pages using this knob may be used when swapping is +still possible to + - prevent disk I/O thrashing under memory pressure; + - improve performance in disk cache-bound tasks under memory pressure. + +Setting it to a high value may result in a early eviction of anonymous pages +into the swap space by attempting to hold the protected amount of clean file +pages in memory. + +The unit of measurement is the percentage of the total memory of the node. + +The default value is 15. + + +clean_min_ratio +================ + +This knob provides *hard* protection of clean file pages. The file pages on the +current node won't be reclaimed under memory pressure when the amount of clean +file pages is below vm.clean_min_ratio. + +Hard protection of clean file pages using this knob may be used to + - prevent disk I/O thrashing under memory pressure even with no free swap space; + - improve performance in disk cache-bound tasks under memory pressure; + - avoid high latency and prevent livelock in near-OOM conditions. + +Setting it to a high value may result in a early out-of-memory condition due to +the inability to reclaim the protected amount of clean file pages when other +types of pages cannot be reclaimed. + +The unit of measurement is the percentage of the total memory of the node. + +The default value is 4. + + compact_memory ============== @@ -980,6 +1044,14 @@ be 133 (x + 2x = 200, 2x = 133.33). At 0, the kernel will not initiate swap until the amount of free and file-backed pages is less than the high watermark in a zone. +This knob has no effect if the amount of clean file pages on the current +node is below vm.clean_low_ratio or vm.clean_min_ratio. In this case, +only anonymous pages can be reclaimed. + +If the number of anonymous pages on the current node is below +vm.anon_min_ratio, then only file pages can be reclaimed with +any vm.swappiness value. + unprivileged_userfaultfd ======================== diff --git a/Makefile b/Makefile index e5509045fe3f..4d6908b5a1a2 100644 --- a/Makefile +++ b/Makefile @@ -862,11 +862,19 @@ KBUILD_CFLAGS += -fno-delete-null-pointer-checks ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE KBUILD_CFLAGS += -O2 KBUILD_RUSTFLAGS += -Copt-level=2 +else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 +KBUILD_CFLAGS += -O3 +KBUILD_RUSTFLAGS += -Copt-level=3 else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE KBUILD_CFLAGS += -Os KBUILD_RUSTFLAGS += -Copt-level=s endif +# Perform swing modulo scheduling immediately before the first scheduling pass. +# This pass looks at innermost loops and reorders their instructions by +# overlapping different iterations. +KBUILD_CFLAGS += $(call cc-option,-fmodulo-sched -fmodulo-sched-allow-regmoves -fivopts -fmodulo-sched) + # Always set `debug-assertions` and `overflow-checks` because their default # depends on `opt-level` and `debug-assertions`, respectively. KBUILD_RUSTFLAGS += -Cdebug-assertions=$(if $(CONFIG_RUST_DEBUG_ASSERTIONS),y,n) @@ -996,10 +1004,10 @@ export CC_FLAGS_SCS endif ifdef CONFIG_LTO_CLANG -ifdef CONFIG_LTO_CLANG_THIN -CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit -else +ifdef CONFIG_LTO_CLANG_FULL CC_FLAGS_LTO := -flto +else +CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit endif CC_FLAGS_LTO += -fvisibility=hidden @@ -1192,7 +1200,7 @@ export ARCH_DRIVERS := $(drivers-y) $(drivers-m) KBUILD_VMLINUX_OBJS := built-in.a $(patsubst %/, %/lib.a, $(filter %/, $(libs-y))) KBUILD_VMLINUX_LIBS := $(filter-out %/, $(libs-y)) -export KBUILD_VMLINUX_LIBS +export KBUILD_VMLINUX_OBJS KBUILD_VMLINUX_LIBS export KBUILD_LDS := arch/$(SRCARCH)/kernel/vmlinux.lds ifdef CONFIG_TRIM_UNUSED_KSYMS @@ -1201,16 +1209,12 @@ ifdef CONFIG_TRIM_UNUSED_KSYMS KBUILD_MODULES := y endif -# '$(AR) mPi' needs 'T' to workaround the bug of llvm-ar <= 14 -quiet_cmd_ar_vmlinux.a = AR $@ - cmd_ar_vmlinux.a = \ - rm -f $@; \ - $(AR) cDPrST $@ $(KBUILD_VMLINUX_OBJS); \ - $(AR) mPiT $$($(AR) t $@ | sed -n 1p) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) +PHONY += vmlinux_a +vmlinux_a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt FORCE + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_a -targets += vmlinux.a -vmlinux.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt FORCE - $(call if_changed,ar_vmlinux.a) +vmlinux.a: vmlinux_a + @: PHONY += vmlinux_o vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS) @@ -1570,6 +1574,7 @@ endif # CONFIG_MODULES CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ + vmlinux.thinlto-index builtin.order \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \ .builtin-dtbs-list .builtin-dtb.S @@ -2011,7 +2016,7 @@ clean: $(clean-dirs) $(call cmd,rmfiles) @find . $(RCS_FIND_IGNORE) \ \( -name '*.[aios]' -o -name '*.rsi' -o -name '*.ko' -o -name '.*.cmd' \ - -o -name '*.ko.*' \ + -o -name '*.ko.*' -o -name '*.o.thinlto.bc' \ -o -name '*.dtb' -o -name '*.dtbo' \ -o -name '*.dtb.S' -o -name '*.dtbo.S' \ -o -name '*.dt.yaml' -o -name 'dtbs-list' \ diff --git a/arch/Kconfig b/arch/Kconfig index a3308a220f86..edf3d9f94592 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -810,6 +810,25 @@ config LTO_CLANG_THIN https://clang.llvm.org/docs/ThinLTO.html If unsure, say Y. + +config LTO_CLANG_THIN_DIST + bool "Clang ThinLTO in distributed mode (EXPERIMENTAL)" + depends on HAS_LTO_CLANG && ARCH_SUPPORTS_LTO_CLANG_THIN + select LTO_CLANG + help + This option enables Clang's ThinLTO in distributed build mode. + In this mode, the linker performs the thin-link, generating + ThinLTO index files. Subsequently, the build system explicitly + invokes ThinLTO backend compilation using these index files + and pre-linked IR objects. The resulting native object files + are with the .thinlto-native.o suffix. + + This build mode offers improved visibility into the ThinLTO + process through explicit subcommand exposure. It also makes + final native object files directly available, benefiting + tools like objtool and kpatch. Additionally, it provides + crucial granular control over back-end options, enabling + module-specific compiler options, and simplifies debugging. endchoice config ARCH_SUPPORTS_AUTOFDO_CLANG diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index f928cf6e3252..d4ce964d9713 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -255,6 +255,11 @@ config CC_HAS_MARCH_NATIVE # usage warnings that only appear wth '-march=native'. depends on CC_IS_GCC || CLANG_VERSION >= 190100 + +choice + prompt "x86_64 Compiler Build Optimization" + default GENERIC_CPU + config X86_NATIVE_CPU bool "Build and optimize for local/native CPU" depends on X86_64 @@ -269,6 +274,47 @@ config X86_NATIVE_CPU If unsure, say N. +config GENERIC_CPU + bool "Generic-x86-64" + depends on X86_64 + help + Generic x86-64 CPU. + Runs equally well on all x86-64 CPUs. + +config MZEN4 + bool "AMD Ryzen 4" + depends on (CC_IS_GCC && GCC_VERSION >= 130000) || (CC_IS_CLANG && CLANG_VERSION >= 160000) + help + Select this for AMD Family 19h Zen 4 processors. + + Enables -march=znver4 + +endchoice + +config X86_64_VERSION + int "x86-64 compiler ISA level" + range 1 4 + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + depends on X86_64 && GENERIC_CPU + help + Specify a specific x86-64 compiler ISA level. + + There are three x86-64 ISA levels that work on top of + the x86-64 baseline, namely: x86-64-v2 and x86-64-v3. + + x86-64-v2 brings support for vector instructions up to Streaming SIMD + Extensions 4.2 (SSE4.2) and Supplemental Streaming SIMD Extensions 3 + (SSSE3), the POPCNT instruction, and CMPXCHG16B. + + x86-64-v3 adds vector instructions up to AVX2, MOVBE, and additional + bit-manipulation instructions. + + x86-64-v4 is not included since the kernel does not use AVX512 instructions + + You can find the best version for your CPU by running one of the following: + /lib/ld-linux-x86-64.so.2 --help | grep supported + /lib64/ld-linux-x86-64.so.2 --help | grep supported + config X86_GENERIC bool "Generic x86 support" depends on X86_32 diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 1913d342969b..82358ed864bb 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -176,10 +176,22 @@ else ifdef CONFIG_X86_NATIVE_CPU KBUILD_CFLAGS += -march=native KBUILD_RUSTFLAGS += -Ctarget-cpu=native -else +endif + +ifdef CONFIG_MZEN4 + KBUILD_CFLAGS += -march=znver4 + KBUILD_RUSTFLAGS += -Ctarget-cpu=znver4 +endif + +ifdef CONFIG_GENERIC_CPU +ifeq ($(CONFIG_X86_64_VERSION),1) KBUILD_CFLAGS += -march=x86-64 -mtune=generic KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64 -Ztune-cpu=generic -endif +else + KBUILD_CFLAGS +=-march=x86-64-v$(CONFIG_X86_64_VERSION) + KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64-v$(CONFIG_X86_64_VERSION) +endif # CONFIG_X86_64_VERSION +endif # CONFIG_GENERIC_CPU KBUILD_CFLAGS += -mno-red-zone KBUILD_CFLAGS += -mcmodel=kernel diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h index b3ab80a03365..5e883b397ff3 100644 --- a/arch/x86/include/asm/pci.h +++ b/arch/x86/include/asm/pci.h @@ -26,6 +26,7 @@ struct pci_sysdata { #if IS_ENABLED(CONFIG_VMD) struct pci_dev *vmd_dev; /* VMD Device if in Intel VMD domain */ #endif + struct pci_dev *nvme_remap_dev; /* AHCI Device if NVME remapped bus */ }; extern int pci_routeirq; @@ -69,6 +70,11 @@ static inline bool is_vmd(struct pci_bus *bus) #define is_vmd(bus) false #endif /* CONFIG_VMD */ +static inline bool is_nvme_remap(struct pci_bus *bus) +{ + return to_pci_sysdata(bus)->nvme_remap_dev != NULL; +} + /* Can be used to override the logic in pci_scan_bus for skipping already-configured bus numbers - to be used for buggy BIOSes or architectures with incomplete PCI setup by the loader */ diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c index ddb798603201..7c20387d8202 100644 --- a/arch/x86/pci/common.c +++ b/arch/x86/pci/common.c @@ -723,12 +723,15 @@ int pci_ext_cfg_avail(void) return 0; } -#if IS_ENABLED(CONFIG_VMD) struct pci_dev *pci_real_dma_dev(struct pci_dev *dev) { +#if IS_ENABLED(CONFIG_VMD) if (is_vmd(dev->bus)) return to_pci_sysdata(dev->bus)->vmd_dev; +#endif + + if (is_nvme_remap(dev->bus)) + return to_pci_sysdata(dev->bus)->nvme_remap_dev; return dev; } -#endif diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched index 27f11320b8d1..e98585dd83e0 100644 --- a/block/Kconfig.iosched +++ b/block/Kconfig.iosched @@ -16,6 +16,20 @@ config MQ_IOSCHED_KYBER synchronous writes, it will self-tune queue depths to achieve that goal. +config MQ_IOSCHED_ADIOS + tristate "Adaptive Deadline I/O scheduler" + default m + help + The Adaptive Deadline I/O Scheduler (ADIOS) is a multi-queue I/O + scheduler with learning-based adaptive latency control. + +config MQ_IOSCHED_DEFAULT_ADIOS + bool "Enable ADIOS I/O scheduler as default MQ I/O scheduler" + depends on MQ_IOSCHED_ADIOS=y + default n + help + Enable the ADIOS I/O scheduler as the default scheduler for MQ I/O. + config IOSCHED_BFQ tristate "BFQ I/O scheduler" select BLK_ICQ diff --git a/block/Makefile b/block/Makefile index c65f4da93702..105b12fd86b8 100644 --- a/block/Makefile +++ b/block/Makefile @@ -22,6 +22,7 @@ obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o obj-$(CONFIG_MQ_IOSCHED_DEADLINE) += mq-deadline.o obj-$(CONFIG_MQ_IOSCHED_KYBER) += kyber-iosched.o +obj-$(CONFIG_MQ_IOSCHED_ADIOS) += adios.o bfq-y := bfq-iosched.o bfq-wf2q.o bfq-cgroup.o obj-$(CONFIG_IOSCHED_BFQ) += bfq.o @@ -36,3 +37,10 @@ obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += blk-crypto.o blk-crypto-profile.o \ blk-crypto-sysfs.o obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o obj-$(CONFIG_BLOCK_HOLDER_DEPRECATED) += holder.o + +all: + make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules + +clean: + make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean + diff --git a/block/adios.c b/block/adios.c new file mode 100644 index 000000000000..bcc90564b9ce --- /dev/null +++ b/block/adios.c @@ -0,0 +1,1881 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Adaptive Deadline I/O Scheduler (ADIOS) + * Copyright (C) 2025 Masahito Suzuki + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "elevator.h" +#include "blk.h" +#include "blk-mq.h" +#include "blk-mq-sched.h" + +#define ADIOS_VERSION "3.0.1" + +/* Request Types: + * + * Tier 0 (Highest Priority): Emergency & System Integrity Requests + * ----------------------------------------------------------------- + * - Target: Requests with the BLK_MQ_INSERT_AT_HEAD flag. + * - Purpose: For critical, non-negotiable operations such as device error + * recovery or flush sequences that must bypass all other scheduling logic. + * - Implementation: Placed in a dedicated, high-priority FIFO queue + * (`prio_queue[0]`) for immediate dispatch. + * + * Tier 1 (High Priority): Data Persistence & Ordering Guarantees + * --------------------------------------------------------------- + * - Target: Requests with integrity-sensitive flags like REQ_FUA or + * REQ_PREFLUSH, typically originating from O_DIRECT I/O. + * - Purpose: To ensure strict ordering and data persistence guarantees, + * preventing data corruption in applications like databases. + * - Implementation: Handled in a separate, secondary FIFO queue + * (`prio_queue[1]`) to ensure they are processed in submission order and + * before any lower-priority requests. + * + * Tier 2 (Medium Priority): Application Responsiveness + * ---------------------------------------------------- + * - Target: Normal synchronous requests (e.g., from standard file reads). + * - Purpose: To ensure correct application behavior for operations that + * depend on sequential I/O completion (e.g., file system mounts) and to + * provide low latency for interactive applications. + * - Implementation: The deadline for these requests is set to their start + * time (`rq->start_time_ns`). This effectively enforces FIFO-like behavior + * within the deadline-sorted red-black tree, preventing out-of-order + * execution of dependent synchronous operations. + * + * Tier 3 (Normal Priority): Background Throughput + * ----------------------------------------------- + * - Target: Asynchronous requests. + * - Purpose: To maximize disk throughput for background tasks where latency + * is not critical. + * - Implementation: These are the only requests where ADIOS's adaptive + * latency prediction model is used. A dynamic deadline is calculated based + * on the predicted I/O latency, allowing for aggressive reordering to + * optimize I/O efficiency. + * + * Dispatch Logic: + * The scheduler always dispatches requests in strict priority order: + * 1. prio_queue[0] (Tier 0) + * 2. prio_queue[1] (Tier 1) + * 3. The deadline-sorted batch queue (which naturally prioritizes Tier 2 + * over Tier 3 due to their calculated deadlines). + */ + +// Global variable to control the latency +static u64 default_global_latency_window = 16000000ULL; +static u64 default_global_latency_window_rotational = 22000000ULL; +// Ratio below which batch queues should be refilled +static u8 default_bq_refill_below_ratio = 20; +// Maximum latency sample to input +static u64 default_lat_model_latency_limit = 500000000ULL; +// Batch ordering strategy +static u64 default_batch_order = 0; +// Flags to control compliance with block layer constraints +static u64 default_compliance_flags = 0x7; + +/* Compliance Flags: + * 0x1: REQ_FUA requests will be handled as Tier-1, strictly prioritized + * 0x2: REQ_PREFLUSH requests will be handled as Tier-1, strictly prioritized + * 0x4: Async requests will not be reordered based on the predicted latency + */ +enum adios_compliance_flags { + ADIOS_CF_PRIO_FUA = 1U << 0, + ADIOS_CF_PRIO_PF = 1U << 1, + ADIOS_CF_FIXORDER = 1U << 2, +}; + +// Dynamic thresholds for shrinkage +static u32 default_lm_shrink_at_kreqs = 5000; +static u32 default_lm_shrink_at_gbytes = 50; +static u32 default_lm_shrink_resist = 2; + +enum adios_optype { + ADIOS_READ = 0, + ADIOS_WRITE = 1, + ADIOS_DISCARD = 2, + ADIOS_OTHER = 3, + ADIOS_OPTYPES = 4, +}; + +// Latency targets for each operation type +static u64 default_latency_target[ADIOS_OPTYPES] = { + [ADIOS_READ] = 2ULL * NSEC_PER_MSEC, + [ADIOS_WRITE] = 2000ULL * NSEC_PER_MSEC, + [ADIOS_DISCARD] = 8000ULL * NSEC_PER_MSEC, + [ADIOS_OTHER] = 0ULL * NSEC_PER_MSEC, +}; + +// Maximum batch size limits for each operation type +static u32 default_batch_limit[ADIOS_OPTYPES] = { + [ADIOS_READ] = 36, + [ADIOS_WRITE] = 72, + [ADIOS_DISCARD] = 1, + [ADIOS_OTHER] = 1, +}; + +enum adios_batch_order { + ADIOS_BO_OPTYPE = 0, + ADIOS_BO_ELEVATOR = 1, +}; + +// Thresholds for latency model control +#define LM_BLOCK_SIZE_THRESHOLD 4096 +#define LM_SAMPLES_THRESHOLD 1024 +#define LM_INTERVAL_THRESHOLD 1500 +#define LM_OUTLIER_PERCENTILE 99 +#define LM_LAT_BUCKET_COUNT 64 + +#define ADIOS_PQ_LEVELS 2 +#define ADIOS_DL_TYPES 2 +#define ADIOS_BQ_PAGES 2 + +static u32 default_dl_prio[ADIOS_DL_TYPES] = {8, 0}; + +// Bit flags for the atomic state variable, indicating which queues have requests. +enum adios_state_flags { + ADIOS_STATE_PQ_0 = 1U << 0, + ADIOS_STATE_PQ_1 = 1U << 1, + ADIOS_STATE_DL_0 = 1U << 2, + ADIOS_STATE_DL_1 = 1U << 3, + ADIOS_STATE_BQ_PAGE_0 = 1U << 4, + ADIOS_STATE_BQ_PAGE_1 = 1U << 5, +}; +#define ADIOS_STATE_PQ 0 +#define ADIOS_STATE_DL 2 +#define ADIOS_STATE_BQ 4 + +// Temporal granularity of the deadline tree node (dl_group) +#define ADIOS_QUANTUM_SHIFT 20 + +#define ADIOS_MAX_INSERTS_PER_LOCK 72 +#define ADIOS_MAX_DELETES_PER_LOCK 24 + +// Structure to hold latency bucket data for small requests +struct latency_bucket_small { + u64 weighted_sum_latency; + u64 sum_of_weights; +}; + +// Structure to hold latency bucket data for large requests +struct latency_bucket_large { + u64 weighted_sum_latency; + u64 weighted_sum_block_size; + u64 sum_of_weights; +}; + +// Structure to hold per-cpu buckets, improving data locality and code clarity. +struct lm_buckets { + struct latency_bucket_small small_bucket[LM_LAT_BUCKET_COUNT]; + struct latency_bucket_large large_bucket[LM_LAT_BUCKET_COUNT]; +}; + +// Structure to hold RCU-protected latency model parameters +struct latency_model_params { + u64 base; + u64 slope; + u64 small_sum_delay; + u64 small_count; + u64 large_sum_delay; + u64 large_sum_bsize; + u64 last_update_jiffies; + struct rcu_head rcu; +}; + +// Structure to hold the latency model context data +struct latency_model { + spinlock_t update_lock; + struct latency_model_params __rcu *params; + + // Per-CPU buckets to avoid lock contention on the completion path + struct lm_buckets __percpu *pcpu_buckets; + + u32 lm_shrink_at_kreqs; + u32 lm_shrink_at_gbytes; + u8 lm_shrink_resist; +}; + +// Adios scheduler data +struct adios_data { + spinlock_t pq_lock; + struct list_head prio_queue[2]; + + struct rb_root_cached dl_tree[2]; + spinlock_t lock; + s64 dl_bias; + s32 dl_prio[2]; + + atomic_t state; + u8 bq_state[ADIOS_BQ_PAGES]; + + void (*insert_request_fn)(struct blk_mq_hw_ctx *, struct request *, + blk_insert_t, struct list_head *); + + u64 global_latency_window; + u64 compliance_flags; + u64 latency_target[ADIOS_OPTYPES]; + u32 batch_limit[ADIOS_OPTYPES]; + u32 batch_actual_max_size[ADIOS_OPTYPES]; + u32 batch_actual_max_total; + u32 async_depth; + u32 lat_model_latency_limit; + u8 bq_refill_below_ratio; + u8 is_rotational; + u8 batch_order; + u8 elv_direction; + sector_t head_pos; + sector_t last_completed_pos; + + bool bq_page; + struct list_head batch_queue[ADIOS_BQ_PAGES][ADIOS_OPTYPES]; + u32 batch_count[ADIOS_BQ_PAGES][ADIOS_OPTYPES]; + u8 bq_batch_order[ADIOS_BQ_PAGES]; + spinlock_t bq_lock; + + struct lm_buckets *aggr_buckets; + + struct latency_model latency_model[ADIOS_OPTYPES]; + struct timer_list update_timer; + + atomic64_t total_pred_lat; + u64 last_completed_time; + + struct kmem_cache *rq_data_pool; + struct kmem_cache *dl_group_pool; + + struct request_queue *queue; +}; + +// List of requests with the same deadline in the deadline-sorted tree +struct dl_group { + struct rb_node node; + struct list_head rqs; + u64 deadline; +} __attribute__((aligned(64))); + +// Structure to hold scheduler-specific data for each request +struct adios_rq_data { + struct list_head *dl_group; + struct list_head dl_node; + + struct request *rq; + u64 deadline; + u64 pred_lat; + u32 block_size; +} __attribute__((aligned(64))); + +static const int adios_prio_to_wmult[40] = { + /* -20 */ 88761, 71755, 56483, 46273, 36291, + /* -15 */ 29154, 23254, 18705, 14949, 11916, + /* -10 */ 9548, 7620, 6100, 4904, 3906, + /* -5 */ 3121, 2501, 1991, 1586, 1277, + /* 0 */ 1024, 820, 655, 526, 423, + /* 5 */ 335, 272, 215, 172, 137, + /* 10 */ 110, 87, 70, 56, 45, + /* 15 */ 36, 29, 23, 18, 15, +}; + +static inline bool compliant(struct adios_data *ad, u32 flag) { + return ad->compliance_flags & flag; +} + +// Count the number of entries in aggregated small buckets +static u64 lm_count_small_entries(struct latency_bucket_small *buckets) { + u64 total_weight = 0; + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) + total_weight += buckets[i].sum_of_weights; + return total_weight; +} + +// Update the small buckets in the latency model from aggregated data +static bool lm_update_small_buckets(struct latency_model *model, + struct latency_model_params *params, + struct latency_bucket_small *buckets, + u64 total_weight, bool count_all) { + u64 sum_latency = 0; + u64 sum_weight = 0; + u64 cumulative_weight = 0, threshold_weight = 0; + u8 outlier_threshold_bucket = 0; + u8 outlier_percentile = LM_OUTLIER_PERCENTILE; + u8 reduction; + + if (count_all) + outlier_percentile = 100; + + // Calculate the threshold weight for outlier detection + threshold_weight = (total_weight * outlier_percentile) / 100; + + // Identify the bucket that corresponds to the outlier threshold + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + cumulative_weight += buckets[i].sum_of_weights; + if (cumulative_weight >= threshold_weight) { + outlier_threshold_bucket = i; + break; + } + } + + // Calculate the average latency, excluding outliers + for (u8 i = 0; i <= outlier_threshold_bucket; i++) { + struct latency_bucket_small *bucket = &buckets[i]; + if (i < outlier_threshold_bucket) { + sum_latency += bucket->weighted_sum_latency; + sum_weight += bucket->sum_of_weights; + } else { + // The threshold bucket's contribution is proportional + u64 remaining_weight = + threshold_weight - (cumulative_weight - bucket->sum_of_weights); + if (bucket->sum_of_weights > 0) { + sum_latency += div_u64(bucket->weighted_sum_latency * + remaining_weight, bucket->sum_of_weights); + sum_weight += remaining_weight; + } + } + } + + // Shrink the model if it reaches at the readjustment threshold + if (params->small_count >= 1000ULL * model->lm_shrink_at_kreqs) { + reduction = model->lm_shrink_resist; + if (params->small_count >> reduction) { + params->small_sum_delay -= params->small_sum_delay >> reduction; + params->small_count -= params->small_count >> reduction; + } + } + + if (!sum_weight) + return false; + + // Accumulate the average latency into the statistics + params->small_sum_delay += sum_latency; + params->small_count += sum_weight; + + return true; +} + +// Count the number of entries in aggregated large buckets +static u64 lm_count_large_entries(struct latency_bucket_large *buckets) { + u64 total_weight = 0; + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) + total_weight += buckets[i].sum_of_weights; + return total_weight; +} + +// Update the large buckets in the latency model from aggregated data +static bool lm_update_large_buckets(struct latency_model *model, + struct latency_model_params *params, + struct latency_bucket_large *buckets, + u64 total_weight, bool count_all) { + s64 sum_latency = 0; + u64 sum_block_size = 0, intercept; + u64 cumulative_weight = 0, threshold_weight = 0; + u64 sum_weight = 0; + u8 outlier_threshold_bucket = 0; + u8 outlier_percentile = LM_OUTLIER_PERCENTILE; + u8 reduction; + + if (count_all) + outlier_percentile = 100; + + // Calculate the threshold weight for outlier detection + threshold_weight = (total_weight * outlier_percentile) / 100; + + // Identify the bucket that corresponds to the outlier threshold + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + cumulative_weight += buckets[i].sum_of_weights; + if (cumulative_weight >= threshold_weight) { + outlier_threshold_bucket = i; + break; + } + } + + // Calculate the average latency and block size, excluding outliers + for (u8 i = 0; i <= outlier_threshold_bucket; i++) { + struct latency_bucket_large *bucket = &buckets[i]; + if (i < outlier_threshold_bucket) { + sum_latency += bucket->weighted_sum_latency; + sum_block_size += bucket->weighted_sum_block_size; + sum_weight += bucket->sum_of_weights; + } else { + // The threshold bucket's contribution is proportional + u64 remaining_weight = + threshold_weight - (cumulative_weight - bucket->sum_of_weights); + if (bucket->sum_of_weights > 0) { + sum_latency += div_u64(bucket->weighted_sum_latency * + remaining_weight, bucket->sum_of_weights); + sum_block_size += div_u64(bucket->weighted_sum_block_size * + remaining_weight, bucket->sum_of_weights); + sum_weight += remaining_weight; + } + } + } + + if (!sum_weight) + return false; + + // Shrink the model if it reaches at the readjustment threshold + if (params->large_sum_bsize >= 0x40000000ULL * model->lm_shrink_at_gbytes) { + reduction = model->lm_shrink_resist; + if (params->large_sum_bsize >> reduction) { + params->large_sum_delay -= params->large_sum_delay >> reduction; + params->large_sum_bsize -= params->large_sum_bsize >> reduction; + } + } + + // Accumulate the average delay into the statistics + intercept = params->base; + if (sum_latency > intercept) + sum_latency -= intercept; + + params->large_sum_delay += sum_latency; + params->large_sum_bsize += sum_block_size; + + return true; +} + +static void reset_buckets(struct lm_buckets *buckets) +{ memset(buckets, 0, sizeof(*buckets)); } + +static void lm_reset_pcpu_buckets(struct latency_model *model) { + int cpu; + for_each_possible_cpu(cpu) + reset_buckets(per_cpu_ptr(model->pcpu_buckets, cpu)); +} + +// Update the latency model parameters and statistics +static void latency_model_update( + struct adios_data *ad, struct latency_model *model) { + u64 now; + u64 small_weight, large_weight; + bool time_elapsed; + bool small_processed = false, large_processed = false; + struct lm_buckets *aggr = ad->aggr_buckets; + struct latency_bucket_small *asb; + struct latency_bucket_large *alb; + struct lm_buckets *pcpu_b; + unsigned long flags; + int cpu; + struct latency_model_params *old_params, *new_params; + + spin_lock_irqsave(&model->update_lock, flags); + + old_params = rcu_dereference_protected(model->params, + lockdep_is_held(&model->update_lock)); + new_params = kmemdup(old_params, sizeof(*new_params), GFP_ATOMIC); + if (!new_params) { + spin_unlock_irqrestore(&model->update_lock, flags); + return; + } + + // Aggregate data from all CPUs and reset per-cpu buckets. + for_each_possible_cpu(cpu) { + pcpu_b = per_cpu_ptr(model->pcpu_buckets, cpu); + + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + if (pcpu_b->small_bucket[i].sum_of_weights) { + asb = &aggr->small_bucket[i]; + asb->sum_of_weights += + pcpu_b->small_bucket[i].sum_of_weights; + asb->weighted_sum_latency += + pcpu_b->small_bucket[i].weighted_sum_latency; + } + if (pcpu_b->large_bucket[i].sum_of_weights) { + alb = &aggr->large_bucket[i]; + alb->sum_of_weights += + pcpu_b->large_bucket[i].sum_of_weights; + alb->weighted_sum_latency += + pcpu_b->large_bucket[i].weighted_sum_latency; + alb->weighted_sum_block_size += + pcpu_b->large_bucket[i].weighted_sum_block_size; + } + } + // Reset per-cpu buckets after aggregating + reset_buckets(pcpu_b); + } + + // Count the number of entries in aggregated buckets + small_weight = lm_count_small_entries(aggr->small_bucket); + large_weight = lm_count_large_entries(aggr->large_bucket); + + // Whether enough time has elapsed since the last update + now = jiffies; + time_elapsed = unlikely(!new_params->base) || + new_params->last_update_jiffies + + msecs_to_jiffies(LM_INTERVAL_THRESHOLD) <= now; + + // Update small buckets + if (small_weight && (time_elapsed || + LM_SAMPLES_THRESHOLD <= small_weight || !new_params->base)) { + small_processed = lm_update_small_buckets(model, new_params, + aggr->small_bucket, small_weight, !new_params->base); + memset(&aggr->small_bucket[0], 0, sizeof(aggr->small_bucket)); + } + // Update large buckets + if (large_weight && (time_elapsed || + LM_SAMPLES_THRESHOLD <= large_weight || !new_params->slope)) { + large_processed = lm_update_large_buckets(model, new_params, + aggr->large_bucket, large_weight, !new_params->slope); + memset(&aggr->large_bucket[0], 0, sizeof(aggr->large_bucket)); + } + + // Update the base parameter if small bucket was processed + if (small_processed && likely(new_params->small_count)) + new_params->base = div_u64(new_params->small_sum_delay, + new_params->small_count); + + // Update the slope parameter if large bucket was processed + if (large_processed && likely(new_params->large_sum_bsize)) + new_params->slope = div_u64(new_params->large_sum_delay, + DIV_ROUND_UP_ULL(new_params->large_sum_bsize, 1024)); + + // Update last updated jiffies if update happened or time has elapsed + if (small_processed || large_processed || time_elapsed) + new_params->last_update_jiffies = now; + + rcu_assign_pointer(model->params, new_params); + spin_unlock_irqrestore(&model->update_lock, flags); + + kfree_rcu(old_params, rcu); +} + +// Determine the bucket index for a given measured and predicted latency +static u8 lm_input_bucket_index(u64 measured, u64 predicted) { + u8 bucket_index; + + if (measured < predicted * 2) + bucket_index = div_u64((measured * 20), predicted); + else if (measured < predicted * 5) + bucket_index = div_u64((measured * 10), predicted) + 20; + else + bucket_index = div_u64((measured * 3), predicted) + 40; + + return bucket_index; +} + +// Input latency data into the latency model +static void latency_model_input(struct adios_data *ad, + struct latency_model *model, + u32 block_size, u64 latency, u64 pred_lat, u32 weight) { + unsigned long flags; + u8 bucket_index; + struct lm_buckets *buckets; + u64 current_base; + struct latency_model_params *params; + + local_irq_save(flags); + buckets = per_cpu_ptr(model->pcpu_buckets, __smp_processor_id()); + + rcu_read_lock(); + params = rcu_dereference(model->params); + current_base = params->base; + rcu_read_unlock(); + + if (block_size <= LM_BLOCK_SIZE_THRESHOLD) { + // Handle small requests + bucket_index = lm_input_bucket_index(latency, current_base ?: 1); + + if (bucket_index >= LM_LAT_BUCKET_COUNT) + bucket_index = LM_LAT_BUCKET_COUNT - 1; + + buckets->small_bucket[bucket_index].sum_of_weights += weight; + buckets->small_bucket[bucket_index].weighted_sum_latency += + latency * weight; + + local_irq_restore(flags); + + if (unlikely(!current_base)) { + latency_model_update(ad, model); + return; + } + } else { + // Handle large requests + if (!current_base || !pred_lat) { + local_irq_restore(flags); + return; + } + + bucket_index = lm_input_bucket_index(latency, pred_lat); + + if (bucket_index >= LM_LAT_BUCKET_COUNT) + bucket_index = LM_LAT_BUCKET_COUNT - 1; + + buckets->large_bucket[bucket_index].sum_of_weights += weight; + buckets->large_bucket[bucket_index].weighted_sum_latency += + latency * weight; + buckets->large_bucket[bucket_index].weighted_sum_block_size += + block_size * weight; + + local_irq_restore(flags); + } +} + +// Predict the latency for a given block size using the latency model +static u64 latency_model_predict(struct latency_model *model, u32 block_size) { + u64 result; + struct latency_model_params *params; + + rcu_read_lock(); + params = rcu_dereference(model->params); + + result = params->base; + if (block_size > LM_BLOCK_SIZE_THRESHOLD) + result += params->slope * + DIV_ROUND_UP_ULL(block_size - LM_BLOCK_SIZE_THRESHOLD, 1024); + + rcu_read_unlock(); + + return result; +} + +// Determine the type of operation based on request flags +static u8 adios_optype(struct request *rq) { + switch (rq->cmd_flags & REQ_OP_MASK) { + case REQ_OP_READ: + return ADIOS_READ; + case REQ_OP_WRITE: + return ADIOS_WRITE; + case REQ_OP_DISCARD: + return ADIOS_DISCARD; + default: + return ADIOS_OTHER; + } +} + +static inline u8 adios_optype_not_read(struct request *rq) { + return (rq->cmd_flags & REQ_OP_MASK) != REQ_OP_READ; +} + +// Helper function to retrieve adios_rq_data from a request +static inline struct adios_rq_data *get_rq_data(struct request *rq) { + return rq->elv.priv[0]; +} + +static inline +void set_adios_state(struct adios_data *ad, u32 shift, u32 idx, bool flag) { + if (flag) + atomic_or(1U << (idx + shift), &ad->state); + else + atomic_andnot(1U << (idx + shift), &ad->state); +} + +static inline u32 get_adios_state(struct adios_data *ad, u32 shift) +{ return (atomic_read(&ad->state) >> shift) & 0x3; } + +// Add a request to the deadline-sorted red-black tree +static void add_to_dl_tree( + struct adios_data *ad, bool dl_idx, struct request *rq) { + struct rb_root_cached *root = &ad->dl_tree[dl_idx]; + struct rb_node **link = &(root->rb_root.rb_node), *parent = NULL; + bool leftmost = true; + struct adios_rq_data *rd = get_rq_data(rq); + struct dl_group *dlg; + u64 deadline; + bool was_empty = RB_EMPTY_ROOT(&root->rb_root); + + /* Tier-2: Synchronous Requests + * - Needs to be FIFO within a same optype + * - Relaxed order between different optypes + * - basically needs to be processed in early time */ + rd->deadline = rq->start_time_ns; + + /* Tier-3: Aynchronous Requests + * - Can be reordered and delayed freely */ + if (!(rq->cmd_flags & REQ_SYNC)) { + rd->deadline += ad->latency_target[adios_optype(rq)]; + if (!compliant(ad, ADIOS_CF_FIXORDER)) + rd->deadline += rd->pred_lat; + } + + // Now quantize the deadline (-> dlg->deadline == RB-Tree key) + deadline = rd->deadline & ~((1ULL << ADIOS_QUANTUM_SHIFT) - 1); + + while (*link) { + dlg = rb_entry(*link, struct dl_group, node); + s64 diff = deadline - dlg->deadline; + + parent = *link; + if (diff < 0) { + link = &((*link)->rb_left); + } else if (diff > 0) { + link = &((*link)->rb_right); + leftmost = false; + } else { // diff == 0 + goto found; + } + } + + dlg = rb_entry_safe(parent, struct dl_group, node); + if (!dlg || dlg->deadline != deadline) { + dlg = kmem_cache_zalloc(ad->dl_group_pool, GFP_ATOMIC); + if (!dlg) + return; + dlg->deadline = deadline; + INIT_LIST_HEAD(&dlg->rqs); + rb_link_node(&dlg->node, parent, link); + rb_insert_color_cached(&dlg->node, root, leftmost); + } +found: + list_add_tail(&rd->dl_node, &dlg->rqs); + rd->dl_group = &dlg->rqs; + + if (was_empty) + set_adios_state(ad, ADIOS_STATE_DL, dl_idx, true); +} + +// Remove a request from the deadline-sorted red-black tree +static void del_from_dl_tree( + struct adios_data *ad, bool dl_idx, struct request *rq) { + struct rb_root_cached *root = &ad->dl_tree[dl_idx]; + struct adios_rq_data *rd = get_rq_data(rq); + struct dl_group *dlg = container_of(rd->dl_group, struct dl_group, rqs); + + list_del_init(&rd->dl_node); + if (list_empty(&dlg->rqs)) { + rb_erase_cached(&dlg->node, root); + kmem_cache_free(ad->dl_group_pool, dlg); + } + rd->dl_group = NULL; + + if (RB_EMPTY_ROOT(&ad->dl_tree[dl_idx].rb_root)) + set_adios_state(ad, ADIOS_STATE_DL, dl_idx, false); +} + +// Remove a request from the scheduler +static void remove_request(struct adios_data *ad, struct request *rq) { + bool dl_idx = adios_optype_not_read(rq); + struct request_queue *q = rq->q; + struct adios_rq_data *rd = get_rq_data(rq); + + list_del_init(&rq->queuelist); + + // We might not be on the rbtree, if we are doing an insert merge + if (rd->dl_group) + del_from_dl_tree(ad, dl_idx, rq); + + elv_rqhash_del(q, rq); + if (q->last_merge == rq) + q->last_merge = NULL; +} + +// Convert a queue depth to the corresponding word depth for shallow allocation +static int to_word_depth(struct blk_mq_hw_ctx *hctx, unsigned int qdepth) { + struct sbitmap_queue *bt = &hctx->sched_tags->bitmap_tags; + const unsigned int nrr = hctx->queue->nr_requests; + + return ((qdepth << bt->sb.shift) + nrr - 1) / nrr; +} + +// We limit the depth of request allocation for asynchronous and write requests +static void adios_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { + struct adios_data *ad = data->q->elevator->elevator_data; + + // Do not throttle synchronous reads + if (op_is_sync(opf) && !op_is_write(opf)) + return; + + data->shallow_depth = to_word_depth(data->hctx, ad->async_depth); +} + +// The number of requests in the queue was notified from the block layer +static void adios_depth_updated(struct blk_mq_hw_ctx *hctx) { + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + struct blk_mq_tags *tags = hctx->sched_tags; + + ad->async_depth = q->nr_requests; + + sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, 1); +} + +// Handle request merging after a merge operation +static void adios_request_merged(struct request_queue *q, struct request *req, + enum elv_merge type) { + bool dl_idx = adios_optype_not_read(req); + struct adios_data *ad = q->elevator->elevator_data; + + // Reposition request in the deadline-sorted tree + del_from_dl_tree(ad, dl_idx, req); + add_to_dl_tree(ad, dl_idx, req); +} + +// Handle merging of requests after one has been merged into another +static void adios_merged_requests(struct request_queue *q, struct request *req, + struct request *next) { + struct adios_data *ad = q->elevator->elevator_data; + + lockdep_assert_held(&ad->lock); + + // kill knowledge of next, this one is a goner + remove_request(ad, next); +} + +// Try to merge a bio into an existing rq before associating it with an rq +static bool adios_bio_merge(struct request_queue *q, struct bio *bio, + unsigned int nr_segs) { + unsigned long flags; + struct adios_data *ad = q->elevator->elevator_data; + struct request *free = NULL; + bool ret; + + if (!spin_trylock_irqsave(&ad->lock, flags)) + return false; + + ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); + spin_unlock_irqrestore(&ad->lock, flags); + + if (free) + blk_mq_free_request(free); + + return ret; +} + +// Insert a request into the scheduler (after Read & Write models stabilized) +static void insert_request_post_stability(struct blk_mq_hw_ctx *hctx, + struct request *rq, blk_insert_t insert_flags, struct list_head *free) { + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + bool dl_idx; + u8 optype = adios_optype(rq); + u8 insert_pq_flags = 0; + + rd->block_size = blk_rq_bytes(rq); + rd->pred_lat = + latency_model_predict(&ad->latency_model[optype], rd->block_size); + + /* Tier-0: BLK_MQ_INSERT_AT_HEAD Requests + * - Needs to be processed ASAP at all costs in any case */ + if (insert_flags & BLK_MQ_INSERT_AT_HEAD) + { insert_pq_flags |= 0x2; } + /* Tier-1: Integrity-sensitive Requests + * - Needs to be FIFO across all optypes */ + if ((compliant(ad, ADIOS_CF_PRIO_FUA) && (rq->cmd_flags & REQ_FUA)) || + (compliant(ad, ADIOS_CF_PRIO_PF ) && (rq->cmd_flags & REQ_PREFLUSH))) + { insert_pq_flags |= 0x1; } + + if (insert_pq_flags) { + u8 pq_idx = !(insert_pq_flags >> 1); + if (rd->pred_lat) + atomic64_add(rd->pred_lat, &ad->total_pred_lat); + scoped_guard(spinlock_irqsave, &ad->pq_lock) { + bool was_empty = list_empty(&ad->prio_queue[pq_idx]); + list_add_tail(&rq->queuelist, &ad->prio_queue[pq_idx]); + if (was_empty) + set_adios_state(ad, ADIOS_STATE_PQ, pq_idx, true); + } + return; + } + + if (blk_mq_sched_try_insert_merge(q, rq, free)) + return; + + dl_idx = adios_optype_not_read(rq); + add_to_dl_tree(ad, dl_idx, rq); + + if (rq_mergeable(rq)) { + elv_rqhash_add(q, rq); + if (!q->last_merge) + q->last_merge = rq; + } +} + +// Insert a request into the scheduler (before Read & Write models stabilizes) +static void insert_request_pre_stability(struct blk_mq_hw_ctx *hctx, + struct request *rq, blk_insert_t insert_flags, struct list_head *free) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + u8 optype = adios_optype(rq); + u8 pq_idx = !(insert_flags & BLK_MQ_INSERT_AT_HEAD); + bool models_stable = false; + + rd->block_size = blk_rq_bytes(rq); + rd->pred_lat = + latency_model_predict(&ad->latency_model[optype], rd->block_size); + + if (rd->pred_lat) + atomic64_add(rd->pred_lat, &ad->total_pred_lat); + + scoped_guard(spinlock_irqsave, &ad->pq_lock) { + bool was_empty = list_empty(&ad->prio_queue[pq_idx]); + list_add_tail(&rq->queuelist, &ad->prio_queue[pq_idx]); + if (was_empty) + set_adios_state(ad, ADIOS_STATE_PQ, pq_idx, true); + } + + rcu_read_lock(); + if (rcu_dereference(ad->latency_model[ADIOS_READ].params)->base > 0 && + rcu_dereference(ad->latency_model[ADIOS_WRITE].params)->base > 0) + models_stable = true; + rcu_read_unlock(); + + if (models_stable) + ad->insert_request_fn = insert_request_post_stability; +} + +// Insert multiple requests into the scheduler +static void adios_insert_requests(struct blk_mq_hw_ctx *hctx, + struct list_head *list, + blk_insert_t insert_flags) { + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + struct request *rq; + bool stop = false; + LIST_HEAD(free); + + do { + scoped_guard(spinlock_irqsave, &ad->lock) + for (int i = 0; i < ADIOS_MAX_INSERTS_PER_LOCK; i++) { + if (list_empty(list)) { + stop = true; + break; + } + rq = list_first_entry(list, struct request, queuelist); + list_del_init(&rq->queuelist); + ad->insert_request_fn(hctx, rq, insert_flags, &free); + }} while (!stop); + + blk_mq_free_requests(&free); +} + +// Prepare a request before it is inserted into the scheduler +static void adios_prepare_request(struct request *rq) { + struct adios_data *ad = rq->q->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + + rq->elv.priv[0] = NULL; + + /* Allocate adios_rq_data from the memory pool */ + rd = kmem_cache_zalloc(ad->rq_data_pool, GFP_ATOMIC); + if (WARN(!rd, "adios_prepare_request: " + "Failed to allocate memory from rq_data_pool. rd is NULL\n")) + return; + + rd->rq = rq; + rq->elv.priv[0] = rd; +} + +static struct adios_rq_data *get_dl_first_rd(struct adios_data *ad, bool idx) { + struct rb_root_cached *root = &ad->dl_tree[idx]; + struct rb_node *first = rb_first_cached(root); + struct dl_group *dl_group = rb_entry(first, struct dl_group, node); + + return list_first_entry(&dl_group->rqs, struct adios_rq_data, dl_node); +} + +// Comparison function for sorting requests by block address +static int cmp_rq_pos(void *priv, + const struct list_head *a, const struct list_head *b) { + struct request *rq_a = list_entry(a, struct request, queuelist); + struct request *rq_b = list_entry(b, struct request, queuelist); + u64 pos_a = blk_rq_pos(rq_a); + u64 pos_b = blk_rq_pos(rq_b); + + return (int)(pos_a > pos_b) - (int)(pos_a < pos_b); +} + +#ifndef list_last_entry_or_null +#define list_last_entry_or_null(ptr, type, member) \ + (!list_empty(ptr) ? list_last_entry(ptr, type, member) : NULL) +#endif + +// Update the elevator direction +static void update_elv_direction(struct adios_data *ad) { + if (!ad->is_rotational) + return; + + bool page = ad->bq_page; + struct list_head *q = &ad->batch_queue[page][1]; + if (ad->bq_batch_order[page] < ADIOS_BO_ELEVATOR || list_empty(q)) { + ad->elv_direction = 0; + return; + } + + // Get first and last request positions in the queue + struct request *rq_a = list_first_entry(q, struct request, queuelist); + struct request *rq_b = list_last_entry (q, struct request, queuelist); + u64 pos_a = blk_rq_pos(rq_a); + u64 pos_b = blk_rq_pos(rq_b); + u64 avg_rq_pos = (pos_a + pos_b) >> 1; + + ad->elv_direction = !!(ad->head_pos > avg_rq_pos); +} + +// Fill the batch queues with requests from the deadline-sorted red-black tree +static bool fill_batch_queues(struct adios_data *ad, u64 tpl) { + struct adios_rq_data *rd; + struct request *rq; + struct list_head *dest_q; + u8 dest_idx; + u64 added_lat = 0; + u32 optype_count[ADIOS_OPTYPES] = {0}; + u32 count = 0; + u8 optype; + bool page = !ad->bq_page, dl_idx, bias_idx, update_bias; + u32 dl_queued; + u8 bq_batch_order; + bool stop = false; + + // Reset batch queue counts for the back page + memset(&ad->batch_count[page], 0, sizeof(ad->batch_count[page])); + + ad->bq_batch_order[page] = + bq_batch_order = ad->batch_order; + + do { + scoped_guard(spinlock_irqsave, &ad->lock) + for (int i = 0; i < ADIOS_MAX_DELETES_PER_LOCK; i++) { + bool has_base = false; + + dl_queued = get_adios_state(ad, ADIOS_STATE_DL); + // Check if there are any requests queued in the deadline tree + if (!dl_queued) { + stop = true; + break; + } + + // Reads if both queues have requests, otherwise pick the non-empty. + dl_idx = dl_queued >> 1; + + // Get the first request from the deadline-sorted tree + rd = get_dl_first_rd(ad, dl_idx); + + bias_idx = ad->dl_bias < 0; + // If read and write requests are queued, choose one based on bias + if (dl_queued == 0x3) { + struct adios_rq_data *trd[2] = {get_dl_first_rd(ad, 0), rd}; + rd = trd[bias_idx]; + + update_bias = (trd[bias_idx]->deadline > trd[!bias_idx]->deadline); + } else + update_bias = (bias_idx == dl_idx); + + rq = rd->rq; + optype = adios_optype(rq); + + rcu_read_lock(); + has_base = + !!rcu_dereference(ad->latency_model[optype].params)->base; + rcu_read_unlock(); + + // Check batch size and total predicted latency + if (count && (!has_base || + ad->batch_count[page][optype] >= ad->batch_limit[optype] || + (tpl + added_lat + rd->pred_lat) > ad->global_latency_window)) { + stop = true; + break; + } + + if (update_bias) { + s64 sign = ((s64)bias_idx << 1) - 1; + if (unlikely(!rd->pred_lat)) + ad->dl_bias = sign; + else + // Adjust the bias based on the predicted latency + ad->dl_bias += sign * (s64)((rd->pred_lat * + adios_prio_to_wmult[ad->dl_prio[bias_idx] + 20]) >> 10); + } + + remove_request(ad, rq); + + // Add request to the corresponding batch queue + dest_idx = (bq_batch_order == ADIOS_BO_OPTYPE || optype == ADIOS_OTHER)? + optype : !!(rd->deadline != rq->start_time_ns); + dest_q = &ad->batch_queue[page][dest_idx]; + list_add_tail(&rq->queuelist, dest_q); + ad->bq_state[page] |= 1U << dest_idx; + ad->batch_count[page][optype]++; + optype_count[optype]++; + added_lat += rd->pred_lat; + count++; + }} while (!stop); + + if (bq_batch_order == ADIOS_BO_ELEVATOR && ad->batch_count[page][1] > 1) + list_sort(NULL, &ad->batch_queue[page][1], cmp_rq_pos); + + if (count) { + if (added_lat) + atomic64_add(added_lat, &ad->total_pred_lat); + + set_adios_state(ad, ADIOS_STATE_BQ, page, true); + + for (optype = 0; optype < ADIOS_OPTYPES; optype++) + if (ad->batch_actual_max_size[optype] < optype_count[optype]) + ad->batch_actual_max_size[optype] = optype_count[optype]; + if (ad->batch_actual_max_total < count) + ad->batch_actual_max_total = count; + } + return count; +} + +// Flip to the next batch queue page +static void flip_bq_page(struct adios_data *ad) { + ad->bq_page = !ad->bq_page; + update_elv_direction(ad); +} + +// Pop a request from the specified index (optype or elevator tier) +static inline struct request *pop_bq_request( + struct adios_data *ad, u8 idx, bool direction) { + bool page = ad->bq_page; + struct list_head *q = &ad->batch_queue[page][idx]; + struct request *rq = direction ? + list_last_entry_or_null (q, struct request, queuelist): + list_first_entry_or_null(q, struct request, queuelist); + if (rq) { + list_del_init(&rq->queuelist); + if (list_empty(q)) + ad->bq_state[page] &= ~(1U << idx); + } + return rq; +} + +static struct request *pop_next_bq_request_optype(struct adios_data *ad) { + u32 bq_state = ad->bq_state[ad->bq_page]; + if (!bq_state) return NULL; + + struct request *rq; + u32 bq_idx = 31 - __builtin_clz(bq_state); + + // Dispatch based on optype (FIFO within each) or single-queue elevator + rq = pop_bq_request(ad, bq_idx, false); + return rq; +} + +static struct request *pop_next_bq_request_elevator(struct adios_data *ad) { + u32 bq_state = ad->bq_state[ad->bq_page]; + if (!bq_state) return NULL; + + struct request *rq; + u32 bq_idx = 31 - __builtin_clz(bq_state); + bool direction = (bq_idx == 1) & ad->elv_direction; + + // Tier-2 (sync) is always high priority + // Tier-3 (async) uses the pre-calculated elevator direction + rq = pop_bq_request(ad, bq_idx, direction); + + /* If batch queue for the sync requests just became empty */ + if (bq_idx == 0 && rq && !(bq_state & 0x1)) + update_elv_direction(ad); + + return rq; +} + +// Returns the state of the other batch queue page +static bool more_bq_ready(struct adios_data *ad, bool page) { + u32 state = get_adios_state(ad, ADIOS_STATE_BQ); + return state & (1U << !page); +} + +// Dispatch a request from the batch queues +static struct request *dispatch_from_bq(struct adios_data *ad) { + struct request *rq; + + guard(spinlock_irqsave)(&ad->bq_lock); + + u64 tpl = atomic64_read(&ad->total_pred_lat); + + if (!more_bq_ready(ad, ad->bq_page) && (!tpl || tpl < div_u64( + ad->global_latency_window * ad->bq_refill_below_ratio, 100))) + fill_batch_queues(ad, tpl); + +again: + // Use the per-page state to decide the dispatch logic, ensuring correctness + rq = (ad->bq_batch_order[ad->bq_page] == ADIOS_BO_ELEVATOR) ? + pop_next_bq_request_elevator(ad): + pop_next_bq_request_optype(ad); + + if (rq) { + bool page = ad->bq_page; + bool is_empty = !ad->bq_state[page]; + if (is_empty) + set_adios_state(ad, ADIOS_STATE_BQ, page, false); + return rq; + } + + // If there's more batch queue page available, flip to it and retry + if (more_bq_ready(ad, ad->bq_page)) { + flip_bq_page(ad); + goto again; + } + + return NULL; +} + +// Dispatch a request from the priority queue +static struct request *dispatch_from_pq(struct adios_data *ad) { + struct request *rq = NULL; + + guard(spinlock_irqsave)(&ad->pq_lock); + u32 pq_state = get_adios_state(ad, ADIOS_STATE_PQ); + u8 pq_idx = pq_state >> 1; + struct list_head *q = &ad->prio_queue[pq_idx]; + + if (unlikely(list_empty(q))) return NULL; + + rq = list_first_entry(q, struct request, queuelist); + list_del_init(&rq->queuelist); + if (list_empty(q)) { + set_adios_state(ad, ADIOS_STATE_PQ, pq_idx, false); + update_elv_direction(ad); + } + return rq; +} + +// Dispatch a request to the hardware queue +static struct request *adios_dispatch_request(struct blk_mq_hw_ctx *hctx) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + struct request *rq; + + rq = dispatch_from_pq(ad); + if (rq) goto found; + rq = dispatch_from_bq(ad); + if (!rq) return NULL; +found: + if (ad->is_rotational) + ad->head_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); + + rq->rq_flags |= RQF_STARTED; + return rq; +} + +// Timer callback function to periodically update latency models +static void update_timer_callback(struct timer_list *t) { + struct adios_data *ad = timer_container_of(ad, t, update_timer); + + for (u8 optype = 0; optype < ADIOS_OPTYPES; optype++) + latency_model_update(ad, &ad->latency_model[optype]); +} + +// Handle the completion of a request +static void adios_completed_request(struct request *rq, u64 now) { + struct adios_data *ad = rq->q->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + + u64 tpl_after = atomic64_sub_return(rd->pred_lat, &ad->total_pred_lat); + u8 optype = adios_optype(rq); + + if (optype == ADIOS_OTHER) { + // Non-positional commands make the head position unpredictable. + // Invalidate our knowledge of the last completed position. + if (ad->is_rotational) + ad->last_completed_pos = 0; + return; + } + + u64 lct = ad->last_completed_time ?: rq->io_start_time_ns; + ad->last_completed_time = (tpl_after) ? now : 0; + + if (!rq->io_start_time_ns || !rd->block_size || unlikely(now < lct)) + return; + + u64 latency = now - lct; + if (latency > ad->lat_model_latency_limit) + return; + + u32 weight = 1; + if (ad->is_rotational) { + sector_t current_pos = blk_rq_pos(rq); + // Only calculate seek distance if we have a valid last position. + if (ad->last_completed_pos > 0) { + u64 seek_distance = abs( + (s64)current_pos - (s64)ad->last_completed_pos); + weight = 65 - __builtin_clzll(seek_distance); + } + // Update (or re-synchronize) our knowledge of the head position. + ad->last_completed_pos = current_pos + blk_rq_sectors(rq); + } + + latency_model_input(ad, &ad->latency_model[optype], + rd->block_size, latency, rd->pred_lat, weight); + timer_reduce(&ad->update_timer, jiffies + msecs_to_jiffies(100)); +} + +// Clean up after a request is finished +static void adios_finish_request(struct request *rq) { + struct adios_data *ad = rq->q->elevator->elevator_data; + + if (rq->elv.priv[0]) { + // Free adios_rq_data back to the memory pool + kmem_cache_free(ad->rq_data_pool, get_rq_data(rq)); + rq->elv.priv[0] = NULL; + } +} + +// Check if there are any requests available for dispatch +static bool adios_has_work(struct blk_mq_hw_ctx *hctx) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + + return atomic_read(&ad->state) != 0; +} + +// Initialize the scheduler-specific data for a hardware queue +static int adios_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { + adios_depth_updated(hctx); + return 0; +} + +// Initialize the scheduler-specific data when initializing the request queue +static int adios_init_sched(struct request_queue *q, struct elevator_type *e) { + struct adios_data *ad; + struct elevator_queue *eq; + int ret = -ENOMEM; + u8 optype = 0; + + eq = elevator_alloc(q, e); + if (!eq) { + pr_err("adios: Failed to allocate the elevator\n"); + return ret; + } + + ad = kzalloc_node(sizeof(*ad), GFP_KERNEL, q->node); + if (!ad) { + pr_err("adios: Failed to create adios_data\n"); + goto put_eq; + } + + // Create a memory pool for adios_rq_data + ad->rq_data_pool = kmem_cache_create("rq_data_pool", + sizeof(struct adios_rq_data), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ad->rq_data_pool) { + pr_err("adios: Failed to create rq_data_pool\n"); + goto free_ad; + } + + /* Create a memory pool for dl_group */ + ad->dl_group_pool = kmem_cache_create("dl_group_pool", + sizeof(struct dl_group), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ad->dl_group_pool) { + pr_err("adios: Failed to create dl_group_pool\n"); + goto destroy_rq_data_pool; + } + + for (int i = 0; i < ADIOS_PQ_LEVELS; i++) + INIT_LIST_HEAD(&ad->prio_queue[i]); + + for (u8 i = 0; i < ADIOS_DL_TYPES; i++) { + ad->dl_tree[i] = RB_ROOT_CACHED; + ad->dl_prio[i] = default_dl_prio[i]; + } + ad->dl_bias = 0; + + for (u8 page = 0; page < ADIOS_BQ_PAGES; page++) + for (optype = 0; optype < ADIOS_OPTYPES; optype++) + INIT_LIST_HEAD(&ad->batch_queue[page][optype]); + + ad->aggr_buckets = kzalloc(sizeof(*ad->aggr_buckets), GFP_KERNEL); + if (!ad->aggr_buckets) { + pr_err("adios: Failed to allocate aggregation buckets\n"); + goto destroy_dl_group_pool; + } + + for (optype = 0; optype < ADIOS_OPTYPES; optype++) { + struct latency_model *model = &ad->latency_model[optype]; + struct latency_model_params *params; + + spin_lock_init(&model->update_lock); + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) { + pr_err("adios: Failed to allocate latency_model_params\n"); + goto free_buckets; + } + params->last_update_jiffies = jiffies; + RCU_INIT_POINTER(model->params, params); + + model->pcpu_buckets = alloc_percpu(struct lm_buckets); + if (!model->pcpu_buckets) { + pr_err("adios: Failed to allocate per-CPU buckets\n"); + kfree(params); + goto free_buckets; + } + + model->lm_shrink_at_kreqs = default_lm_shrink_at_kreqs; + model->lm_shrink_at_gbytes = default_lm_shrink_at_gbytes; + model->lm_shrink_resist = default_lm_shrink_resist; + } + + for (optype = 0; optype < ADIOS_OPTYPES; optype++) { + ad->latency_target[optype] = default_latency_target[optype]; + ad->batch_limit[optype] = default_batch_limit[optype]; + } + + eq->elevator_data = ad; + + ad->is_rotational = !!(q->limits.features & BLK_FEAT_ROTATIONAL); + ad->global_latency_window = (ad->is_rotational)? + default_global_latency_window_rotational: + default_global_latency_window; + ad->bq_refill_below_ratio = default_bq_refill_below_ratio; + ad->lat_model_latency_limit = default_lat_model_latency_limit; + ad->batch_order = default_batch_order; + ad->compliance_flags = default_compliance_flags; + + ad->insert_request_fn = insert_request_pre_stability; + + atomic_set(&ad->state, 0); + + spin_lock_init(&ad->lock); + spin_lock_init(&ad->pq_lock); + spin_lock_init(&ad->bq_lock); + + timer_setup(&ad->update_timer, update_timer_callback, 0); + + /* We dispatch from request queue wide instead of hw queue */ + blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); + + ad->queue = q; + blk_stat_enable_accounting(q); + + q->elevator = eq; + return 0; + +free_buckets: + pr_err("adios: Failed to allocate per-cpu buckets\n"); + while (optype-- > 0) { + struct latency_model *prev_model = &ad->latency_model[optype]; + kfree(rcu_access_pointer(prev_model->params)); + free_percpu(prev_model->pcpu_buckets); + } + kfree(ad->aggr_buckets); +destroy_dl_group_pool: + kmem_cache_destroy(ad->dl_group_pool); +destroy_rq_data_pool: + kmem_cache_destroy(ad->rq_data_pool); +free_ad: + kfree(ad); +put_eq: + kobject_put(&eq->kobj); + return ret; +} + +// Clean up and free resources when exiting the scheduler +static void adios_exit_sched(struct elevator_queue *e) { + struct adios_data *ad = e->elevator_data; + + timer_shutdown_sync(&ad->update_timer); + + for (int i = 0; i < 2; i++) + WARN_ON_ONCE(!list_empty(&ad->prio_queue[i])); + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + struct latency_model *model = &ad->latency_model[i]; + struct latency_model_params *params = rcu_access_pointer(model->params); + + RCU_INIT_POINTER(model->params, NULL); + kfree_rcu(params, rcu); + + free_percpu(model->pcpu_buckets); + } + + synchronize_rcu(); + + kfree(ad->aggr_buckets); + + if (ad->rq_data_pool) + kmem_cache_destroy(ad->rq_data_pool); + + if (ad->dl_group_pool) + kmem_cache_destroy(ad->dl_group_pool); + + blk_stat_disable_accounting(ad->queue); + + kfree(ad); +} + +static void sideload_latency_model( + struct latency_model *model, u64 base, u64 slope) { + struct latency_model_params *old_params, *new_params; + unsigned long flags; + + new_params = kzalloc(sizeof(*new_params), GFP_KERNEL); + if (!new_params) + return; + + spin_lock_irqsave(&model->update_lock, flags); + + old_params = rcu_dereference_protected(model->params, + lockdep_is_held(&model->update_lock)); + + new_params->last_update_jiffies = jiffies; + + // Initialize base and its statistics as a single sample. + new_params->base = base; + new_params->small_sum_delay = base; + new_params->small_count = 1; + + // Initialize slope and its statistics as a single sample. + new_params->slope = slope; + new_params->large_sum_delay = slope; + new_params->large_sum_bsize = 1024; /* Corresponds to 1 KiB */ + + lm_reset_pcpu_buckets(model); + + rcu_assign_pointer(model->params, new_params); + spin_unlock_irqrestore(&model->update_lock, flags); + + kfree_rcu(old_params, rcu); +} + +// Define sysfs attributes for operation types +#define SYSFS_OPTYPE_DECL(name, optype) \ +static ssize_t adios_lat_model_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + struct latency_model *model = &ad->latency_model[optype]; \ + struct latency_model_params *params; \ + ssize_t len = 0; \ + u64 base, slope; \ + rcu_read_lock(); \ + params = rcu_dereference(model->params); \ + base = params->base; \ + slope = params->slope; \ + rcu_read_unlock(); \ + len += sprintf(page, "base : %llu ns\n", base); \ + len += sprintf(page + len, "slope: %llu ns/KiB\n", slope); \ + return len; \ +} \ +static ssize_t adios_lat_model_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + struct latency_model *model = &ad->latency_model[optype]; \ + u64 base, slope; \ + int ret; \ + ret = sscanf(page, "%llu %llu", &base, &slope); \ + if (ret != 2) \ + return -EINVAL; \ + sideload_latency_model(model, base, slope); \ + reset_buckets(ad->aggr_buckets); \ + return count; \ +} \ +static ssize_t adios_lat_target_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%llu\n", ad->latency_target[optype]); \ +} \ +static ssize_t adios_lat_target_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long nsec; \ + int ret; \ + ret = kstrtoul(page, 10, &nsec); \ + if (ret) \ + return ret; \ + sideload_latency_model(&ad->latency_model[optype], 0, 0); \ + ad->latency_target[optype] = nsec; \ + return count; \ +} \ +static ssize_t adios_batch_limit_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%u\n", ad->batch_limit[optype]); \ +} \ +static ssize_t adios_batch_limit_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + unsigned long max_batch; \ + int ret; \ + ret = kstrtoul(page, 10, &max_batch); \ + if (ret || max_batch == 0) \ + return -EINVAL; \ + struct adios_data *ad = e->elevator_data; \ + ad->batch_limit[optype] = max_batch; \ + return count; \ +} + +SYSFS_OPTYPE_DECL(read, ADIOS_READ); +SYSFS_OPTYPE_DECL(write, ADIOS_WRITE); +SYSFS_OPTYPE_DECL(discard, ADIOS_DISCARD); + +// Show the maximum batch size actually achieved for each operation type +static ssize_t adios_batch_actual_max_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + u32 total_count, read_count, write_count, discard_count; + + total_count = ad->batch_actual_max_total; + read_count = ad->batch_actual_max_size[ADIOS_READ]; + write_count = ad->batch_actual_max_size[ADIOS_WRITE]; + discard_count = ad->batch_actual_max_size[ADIOS_DISCARD]; + + return sprintf(page, + "Total : %u\nDiscard: %u\nRead : %u\nWrite : %u\n", + total_count, discard_count, read_count, write_count); +} + +#define SYSFS_ULL_DECL(field, min_val, max_val) \ +static ssize_t adios_##field##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%llu\n", ad->field); \ +} \ +static ssize_t adios_##field##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long val; \ + int ret; \ + ret = kstrtoul(page, 10, &val); \ + if (ret || val < (min_val) || val > (max_val)) \ + return -EINVAL; \ + ad->field = val; \ + return count; \ +} + +SYSFS_ULL_DECL(global_latency_window, 0, ULLONG_MAX) +SYSFS_ULL_DECL(compliance_flags, 0, ULLONG_MAX) + +#define SYSFS_INT_DECL(field, min_val, max_val) \ +static ssize_t adios_##field##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%d\n", ad->field); \ +} \ +static ssize_t adios_##field##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + int val; \ + int ret; \ + ret = kstrtoint(page, 10, &val); \ + if (ret || val < (min_val) || val > (max_val)) \ + return -EINVAL; \ + ad->field = val; \ + return count; \ +} + +SYSFS_INT_DECL(bq_refill_below_ratio, 0, 100) +SYSFS_INT_DECL(lat_model_latency_limit, 0, 2*NSEC_PER_SEC) +SYSFS_INT_DECL(batch_order, ADIOS_BO_OPTYPE, !!ad->is_rotational) + +// Show the read priority +static ssize_t adios_read_priority_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + return sprintf(page, "%d\n", ad->dl_prio[0]); +} + +// Set the read priority +static ssize_t adios_read_priority_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + int prio; + int ret; + + ret = kstrtoint(page, 10, &prio); + if (ret || prio < -20 || prio > 19) + return -EINVAL; + + guard(spinlock_irqsave)(&ad->lock); + ad->dl_prio[0] = prio; + ad->dl_bias = 0; + + return count; +} + +// Reset batch queue statistics +static ssize_t adios_reset_bq_stats_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + unsigned long val; + int ret; + + ret = kstrtoul(page, 10, &val); + if (ret || val != 1) + return -EINVAL; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) + ad->batch_actual_max_size[i] = 0; + + ad->batch_actual_max_total = 0; + + return count; +} + +// Reset the latency model parameters or load them from user input +static ssize_t adios_reset_lat_model_store( + struct elevator_queue *e, const char *page, size_t count) +{ + struct adios_data *ad = e->elevator_data; + struct latency_model *model; + int ret; + + /* + * Differentiate between two modes based on input format: + * 1. "1": Fully reset the model (backward compatibility). + * 2. "R_base R_slope W_base W_slope D_base D_slope": Load values. + */ + if (!strchr(page, ' ')) { + // Mode 1: Full reset. + unsigned long val; + + ret = kstrtoul(page, 10, &val); + if (ret || val != 1) + return -EINVAL; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + model = &ad->latency_model[i]; + sideload_latency_model(model, 0, 0); + } + } else { + // Mode 2: Load initial values for all latency models. + u64 params[3][2]; /* 0:base, 1:slope for R, W, D */ + + ret = sscanf(page, "%llu %llu %llu %llu %llu %llu", + ¶ms[ADIOS_READ ][0], ¶ms[ADIOS_READ ][1], + ¶ms[ADIOS_WRITE ][0], ¶ms[ADIOS_WRITE ][1], + ¶ms[ADIOS_DISCARD][0], ¶ms[ADIOS_DISCARD][1]); + + if (ret != 6) + return -EINVAL; + + for (u8 i = ADIOS_READ; i <= ADIOS_DISCARD; i++) { + model = &ad->latency_model[i]; + sideload_latency_model(model, params[i][0], params[i][1]); + } + } + reset_buckets(ad->aggr_buckets); + + return count; +} + +// Show the ADIOS version +static ssize_t adios_version_show(struct elevator_queue *e, char *page) { + return sprintf(page, "%s\n", ADIOS_VERSION); +} + +// Define sysfs attributes for dynamic thresholds +#define SHRINK_THRESHOLD_ATTR_RW(name, model_field, min_value, max_value) \ +static ssize_t adios_shrink_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long val; \ + int ret; \ + ret = kstrtoul(page, 10, &val); \ + if (ret || val < min_value || val > max_value) \ + return -EINVAL; \ + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { \ + struct latency_model *model = &ad->latency_model[i]; \ + unsigned long flags; \ + spin_lock_irqsave(&model->update_lock, flags); \ + model->model_field = val; \ + spin_unlock_irqrestore(&model->update_lock, flags); \ + } \ + return count; \ +} \ +static ssize_t adios_shrink_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + u32 val = 0; \ + unsigned long flags; \ + struct latency_model *model = &ad->latency_model[0]; \ + spin_lock_irqsave(&model->update_lock, flags); \ + val = model->model_field; \ + spin_unlock_irqrestore(&model->update_lock, flags); \ + return sprintf(page, "%u\n", val); \ +} + +SHRINK_THRESHOLD_ATTR_RW(at_kreqs, lm_shrink_at_kreqs, 1, 100000) +SHRINK_THRESHOLD_ATTR_RW(at_gbytes, lm_shrink_at_gbytes, 1, 1000) +SHRINK_THRESHOLD_ATTR_RW(resist, lm_shrink_resist, 1, 3) + +// Define sysfs attributes +#define AD_ATTR(name, show_func, store_func) \ + __ATTR(name, 0644, show_func, store_func) +#define AD_ATTR_RW(name) \ + __ATTR(name, 0644, adios_##name##_show, adios_##name##_store) +#define AD_ATTR_RO(name) \ + __ATTR(name, 0444, adios_##name##_show, NULL) +#define AD_ATTR_WO(name) \ + __ATTR(name, 0200, NULL, adios_##name##_store) + +// Define sysfs attributes for ADIOS scheduler +static struct elv_fs_entry adios_sched_attrs[] = { + AD_ATTR_RO(batch_actual_max), + AD_ATTR_RW(bq_refill_below_ratio), + AD_ATTR_RW(global_latency_window), + AD_ATTR_RW(lat_model_latency_limit), + AD_ATTR_RW(batch_order), + AD_ATTR_RW(compliance_flags), + + AD_ATTR_RW(batch_limit_read), + AD_ATTR_RW(batch_limit_write), + AD_ATTR_RW(batch_limit_discard), + + AD_ATTR_RW(lat_model_read), + AD_ATTR_RW(lat_model_write), + AD_ATTR_RW(lat_model_discard), + + AD_ATTR_RW(lat_target_read), + AD_ATTR_RW(lat_target_write), + AD_ATTR_RW(lat_target_discard), + + AD_ATTR_RW(shrink_at_kreqs), + AD_ATTR_RW(shrink_at_gbytes), + AD_ATTR_RW(shrink_resist), + + AD_ATTR_RW(read_priority), + + AD_ATTR_WO(reset_bq_stats), + AD_ATTR_WO(reset_lat_model), + AD_ATTR(adios_version, adios_version_show, NULL), + + __ATTR_NULL +}; + +// Define the ADIOS scheduler type +static struct elevator_type mq_adios = { + .ops = { + .next_request = elv_rb_latter_request, + .former_request = elv_rb_former_request, + .limit_depth = adios_limit_depth, + .depth_updated = adios_depth_updated, + .request_merged = adios_request_merged, + .requests_merged = adios_merged_requests, + .bio_merge = adios_bio_merge, + .insert_requests = adios_insert_requests, + .prepare_request = adios_prepare_request, + .dispatch_request = adios_dispatch_request, + .completed_request = adios_completed_request, + .finish_request = adios_finish_request, + .has_work = adios_has_work, + .init_hctx = adios_init_hctx, + .init_sched = adios_init_sched, + .exit_sched = adios_exit_sched, + }, + .elevator_attrs = adios_sched_attrs, + .elevator_name = "adios", + .elevator_owner = THIS_MODULE, +}; +MODULE_ALIAS("mq-adios-iosched"); + +#define ADIOS_PROGNAME "Adaptive Deadline I/O Scheduler" +#define ADIOS_AUTHOR "Masahito Suzuki" + +// Initialize the ADIOS scheduler module +static int __init adios_init(void) { + printk(KERN_INFO "%s %s by %s\n", + ADIOS_PROGNAME, ADIOS_VERSION, ADIOS_AUTHOR); + return elv_register(&mq_adios); +} + +// Exit the ADIOS scheduler module +static void __exit adios_exit(void) { + elv_unregister(&mq_adios); +} + +module_init(adios_init); +module_exit(adios_exit); + +MODULE_AUTHOR(ADIOS_AUTHOR); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION(ADIOS_PROGNAME); \ No newline at end of file diff --git a/block/elevator.c b/block/elevator.c index fe96c6f4753c..7b4f2913841f 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -752,6 +752,21 @@ void elevator_set_default(struct request_queue *q) if (q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT) return; +#ifdef CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + ctx.name = "adios"; +#else // !CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + bool is_sq = q->nr_hw_queues == 1 || blk_mq_is_shared_tags(q->tag_set->flags); +#ifdef CONFIG_CACHY +#ifdef CONFIG_IOSCHED_BFQ + if (is_sq) + ctx.name = "bfq"; +#endif /* CONFIG_IOSCHED_BFQ */ +#else + if (!is_sq) + return; +#endif /* CONFIG_CACHY */ +#endif /* CONFIG_MQ_IOSCHED_DEFAULT_ADIOS */ + /* * For single queue devices, default to using mq-deadline. If we * have multiple queues or mq-deadline is not available, default @@ -761,13 +776,10 @@ void elevator_set_default(struct request_queue *q) if (!e) return; - if ((q->nr_hw_queues == 1 || - blk_mq_is_shared_tags(q->tag_set->flags))) { - err = elevator_change(q, &ctx); - if (err < 0) - pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", - ctx.name, err); - } + err = elevator_change(q, &ctx); + if (err < 0) + pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", + ctx.name, err); elevator_put(e); } diff --git a/drivers/Makefile b/drivers/Makefile index b5749cf67044..5beba9f57254 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -64,14 +64,8 @@ obj-y += char/ # iommu/ comes before gpu as gpu are using iommu controllers obj-y += iommu/ -# gpu/ comes after char for AGP vs DRM startup and after iommu -obj-y += gpu/ - obj-$(CONFIG_CONNECTOR) += connector/ -# i810fb depends on char/agp/ -obj-$(CONFIG_FB_I810) += video/fbdev/i810/ - obj-$(CONFIG_PARPORT) += parport/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ @@ -83,6 +77,13 @@ obj-y += macintosh/ obj-y += scsi/ obj-y += nvme/ obj-$(CONFIG_ATA) += ata/ + +# gpu/ comes after char for AGP vs DRM startup and after iommu +obj-y += gpu/ + +# i810fb depends on char/agp/ +obj-$(CONFIG_FB_I810) += video/fbdev/i810/ + obj-$(CONFIG_TARGET_CORE) += target/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 4f69da480e86..ca45f2aec30d 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1656,7 +1656,7 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance) } #endif -static void ahci_remap_check(struct pci_dev *pdev, int bar, +static int ahci_remap_check(struct pci_dev *pdev, int bar, struct ahci_host_priv *hpriv) { int i; @@ -1669,7 +1669,7 @@ static void ahci_remap_check(struct pci_dev *pdev, int bar, pci_resource_len(pdev, bar) < SZ_512K || bar != AHCI_PCI_BAR_STANDARD || !(readl(hpriv->mmio + AHCI_VSCAP) & 1)) - return; + return 0; cap = readq(hpriv->mmio + AHCI_REMAP_CAP); for (i = 0; i < AHCI_MAX_REMAP; i++) { @@ -1684,18 +1684,11 @@ static void ahci_remap_check(struct pci_dev *pdev, int bar, } if (!hpriv->remapped_nvme) - return; - - dev_warn(&pdev->dev, "Found %u remapped NVMe devices.\n", - hpriv->remapped_nvme); - dev_warn(&pdev->dev, - "Switch your BIOS from RAID to AHCI mode to use them.\n"); + return 0; - /* - * Don't rely on the msi-x capability in the remap case, - * share the legacy interrupt across ahci and remapped devices. - */ - hpriv->flags |= AHCI_HFLAG_NO_MSI; + /* Abort probe, allowing intel-nvme-remap to step in when available */ + dev_info(&pdev->dev, "Device will be handled by intel-nvme-remap.\n"); + return -ENODEV; } static int ahci_get_irq_vector(struct ata_host *host, int port) @@ -1949,7 +1942,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; /* detect remapped nvme devices */ - ahci_remap_check(pdev, ahci_pci_bar, hpriv); + rc = ahci_remap_check(pdev, ahci_pci_bar, hpriv); + if (rc) + return rc; sysfs_add_file_to_group(&pdev->dev.kobj, &dev_attr_remapped_nvme.attr, diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 2c5c228408bf..918e2bebfe78 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -9,7 +9,6 @@ config X86_INTEL_PSTATE select ACPI_PROCESSOR if ACPI select ACPI_CPPC_LIB if X86_64 && ACPI && SCHED_MC_PRIO select CPU_FREQ_GOV_PERFORMANCE - select CPU_FREQ_GOV_SCHEDUTIL if SMP help This driver provides a P state for Intel core processors. The driver implements an internal governor and will become @@ -39,7 +38,6 @@ config X86_AMD_PSTATE depends on X86 && ACPI select ACPI_PROCESSOR select ACPI_CPPC_LIB if X86_64 - select CPU_FREQ_GOV_SCHEDUTIL if SMP help This driver adds a CPUFreq driver which utilizes a fine grain processor performance frequency control range instead of legacy diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 06a1c7dd081f..5a75ec5d7e03 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3949,6 +3949,8 @@ static int __init intel_pstate_setup(char *str) if (!strcmp(str, "disable")) no_load = 1; + else if (!strcmp(str, "enable")) + no_load = 0; else if (!strcmp(str, "active")) default_driver = &intel_pstate; else if (!strcmp(str, "passive")) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index e1d79f483049..b9de7fc6095d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -163,6 +163,7 @@ struct amdgpu_watchdog_timer { */ extern int amdgpu_modeset; extern unsigned int amdgpu_vram_limit; +extern int amdgpu_ignore_min_pcap; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size; extern int amdgpu_gtt_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 4db92e0a60da..a5db963da610 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -147,6 +147,7 @@ enum AMDGPU_DEBUG_MASK { }; unsigned int amdgpu_vram_limit = UINT_MAX; +int amdgpu_ignore_min_pcap = 0; /* do not ignore by default */ int amdgpu_vis_vram_limit; int amdgpu_gart_size = -1; /* auto */ int amdgpu_gtt_size = -1; /* auto */ @@ -268,6 +269,15 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .period = 0x0, /* default to 0x0 (timeout disable) */ }; +/** + * DOC: ignore_min_pcap (int) + * Ignore the minimum power cap. + * Useful on graphics cards where the minimum power cap is very high. + * The default is 0 (Do not ignore). + */ +MODULE_PARM_DESC(ignore_min_pcap, "Ignore the minimum power cap"); +module_param_named(ignore_min_pcap, amdgpu_ignore_min_pcap, int, 0600); + /** * DOC: vramlimit (int) * Restrict the total amount of VRAM in MiB for testing. The default is 0 (Use full VRAM). diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index abd3b6564373..46937e6fa78d 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -56,4 +56,10 @@ config DRM_AMD_SECURE_DISPLAY This option enables the calculation of crc of specific region via debugfs. Cooperate with specific DMCU FW. +config AMD_PRIVATE_COLOR + bool "Enable KMS color management by AMD for AMD" + default n + help + This option extends the KMS color management API with AMD driver-specific properties to enhance the color management support on AMD Steam Deck. + endmenu diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 2d94fec5b545..bf984bba62ec 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -4652,7 +4652,7 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) return r; } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR if (amdgpu_dm_create_color_properties(adev)) { dc_state_release(state->context); kfree(state); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index ebabfe3a512f..4d3ebcaacca1 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -97,7 +97,7 @@ static inline struct fixed31_32 amdgpu_dm_fixpt_from_s3132(__u64 x) return val; } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR /* Pre-defined Transfer Functions (TF) * * AMD driver supports pre-defined mathematical functions for transferring diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 45feb404b097..ee8672919a05 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -491,7 +491,7 @@ static int amdgpu_dm_crtc_late_register(struct drm_crtc *crtc) } #endif -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR /** * dm_crtc_additional_color_mgmt - enable additional color properties * @crtc: DRM CRTC @@ -573,7 +573,7 @@ static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = { #if defined(CONFIG_DEBUG_FS) .late_register = amdgpu_dm_crtc_late_register, #endif -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR .atomic_set_property = amdgpu_dm_atomic_crtc_set_property, .atomic_get_property = amdgpu_dm_atomic_crtc_get_property, #endif @@ -770,7 +770,7 @@ int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm, drm_mode_crtc_set_gamma_size(&acrtc->base, MAX_COLOR_LEGACY_LUT_ENTRIES); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR dm_crtc_additional_color_mgmt(&acrtc->base); #endif return 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index b7c6e8d13435..fc8dda5df287 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1601,7 +1601,7 @@ static void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane, drm_atomic_helper_plane_destroy_state(plane, state); } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR static void dm_atomic_plane_attach_color_mgmt_properties(struct amdgpu_display_manager *dm, struct drm_plane *plane) @@ -1792,7 +1792,7 @@ static const struct drm_plane_funcs dm_plane_funcs = { .atomic_duplicate_state = amdgpu_dm_plane_drm_plane_duplicate_state, .atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state, .format_mod_supported = amdgpu_dm_plane_format_mod_supported, -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR .atomic_set_property = dm_atomic_plane_set_property, .atomic_get_property = dm_atomic_plane_get_property, #endif @@ -1888,7 +1888,7 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, else drm_plane_helper_add(plane, &dm_plane_helper_funcs); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR dm_atomic_plane_attach_color_mgmt_properties(dm, plane); #endif /* Create (reset) the plane state */ diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 39ee81085088..a2df219060dc 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -3073,6 +3073,9 @@ static ssize_t amdgpu_hwmon_show_power_cap_min(struct device *dev, struct device_attribute *attr, char *buf) { + if (amdgpu_ignore_min_pcap) + return sysfs_emit(buf, "%i\n", 0); + return amdgpu_hwmon_show_power_cap_generic(dev, attr, buf, PP_PWR_LIMIT_MIN); } diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 26b8e232f858..99f234e488c0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -2870,7 +2870,10 @@ int smu_get_power_limit(void *handle, *limit = smu->max_power_limit; break; case SMU_PPT_LIMIT_MIN: - *limit = smu->min_power_limit; + if (amdgpu_ignore_min_pcap) + *limit = 0; + else + *limit = smu->min_power_limit; break; default: return -EINVAL; @@ -2894,7 +2897,14 @@ static int smu_set_power_limit(void *handle, uint32_t limit) if (smu->ppt_funcs->set_power_limit) return smu->ppt_funcs->set_power_limit(smu, limit_type, limit); - if ((limit > smu->max_power_limit) || (limit < smu->min_power_limit)) { + if (amdgpu_ignore_min_pcap) { + if ((limit > smu->max_power_limit)) { + dev_err(smu->adev->dev, + "New power limit (%d) is over the max allowed %d\n", + limit, smu->max_power_limit); + return -EINVAL; + } + } else if ((limit > smu->max_power_limit) || (limit < smu->min_power_limit)) { dev_err(smu->adev->dev, "New power limit (%d) is out of range [%d,%d]\n", limit, smu->min_power_limit, smu->max_power_limit); diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_client.c b/drivers/hid/amd-sfh-hid/amd_sfh_client.c index 3438d392920f..0f2cbae39b2b 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_client.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_client.c @@ -146,6 +146,8 @@ static const char *get_sensor_name(int idx) return "gyroscope"; case mag_idx: return "magnetometer"; + case op_idx: + return "operating-mode"; case als_idx: case ACS_IDX: /* ambient color sensor */ return "ALS"; @@ -243,6 +245,20 @@ int amd_sfh_hid_client_init(struct amd_mp2_dev *privdata) rc = -ENOMEM; goto cleanup; } + + if (cl_data->sensor_idx[i] == op_idx) { + info.period = AMD_SFH_IDLE_LOOP; + info.sensor_idx = cl_data->sensor_idx[i]; + info.dma_address = cl_data->sensor_dma_addr[i]; + mp2_ops->start(privdata, info); + cl_data->sensor_sts[i] = amd_sfh_wait_for_response(privdata, + cl_data->sensor_idx[i], + SENSOR_ENABLED); + if (cl_data->sensor_sts[i] == SENSOR_ENABLED) + cl_data->is_any_sensor_enabled = true; + continue; + } + cl_data->sensor_sts[i] = SENSOR_DISABLED; cl_data->sensor_requested_cnt[i] = 0; cl_data->cur_hid_dev = i; @@ -303,6 +319,13 @@ int amd_sfh_hid_client_init(struct amd_mp2_dev *privdata) for (i = 0; i < cl_data->num_hid_devices; i++) { cl_data->cur_hid_dev = i; + if (cl_data->sensor_idx[i] == op_idx) { + dev_dbg(dev, "sid 0x%x (%s) status 0x%x\n", + cl_data->sensor_idx[i], get_sensor_name(cl_data->sensor_idx[i]), + cl_data->sensor_sts[i]); + continue; + } + if (cl_data->sensor_sts[i] == SENSOR_ENABLED) { rc = amdtp_hid_probe(i, cl_data); if (rc) diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_hid.h b/drivers/hid/amd-sfh-hid/amd_sfh_hid.h index 1c91be8daedd..7452b0302953 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_hid.h +++ b/drivers/hid/amd-sfh-hid/amd_sfh_hid.h @@ -11,7 +11,7 @@ #ifndef AMDSFH_HID_H #define AMDSFH_HID_H -#define MAX_HID_DEVICES 6 +#define MAX_HID_DEVICES 7 #define AMD_SFH_HID_VENDOR 0x1022 #define AMD_SFH_HID_PRODUCT 0x0001 diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index 1c1fd63330c9..2983af969579 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -29,6 +29,7 @@ #define ACEL_EN BIT(0) #define GYRO_EN BIT(1) #define MAGNO_EN BIT(2) +#define OP_EN BIT(15) #define HPD_EN BIT(16) #define ALS_EN BIT(19) #define ACS_EN BIT(22) @@ -232,6 +233,9 @@ int amd_mp2_get_sensor_num(struct amd_mp2_dev *privdata, u8 *sensor_id) if (MAGNO_EN & activestatus) sensor_id[num_of_sensors++] = mag_idx; + if (OP_EN & activestatus) + sensor_id[num_of_sensors++] = op_idx; + if (ALS_EN & activestatus) sensor_id[num_of_sensors++] = als_idx; diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h index 05e400a4a83e..2eb61f4e8434 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h @@ -79,6 +79,7 @@ enum sensor_idx { accel_idx = 0, gyro_idx = 1, mag_idx = 2, + op_idx = 15, als_idx = 19 }; diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index b5cbb57ee5f6..a0f7fa1518c6 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -46,6 +46,7 @@ struct evdev_client { struct fasync_struct *fasync; struct evdev *evdev; struct list_head node; + struct rcu_head rcu; enum input_clock_type clk_type; bool revoked; unsigned long *evmasks[EV_CNT]; @@ -368,13 +369,22 @@ static void evdev_attach_client(struct evdev *evdev, spin_unlock(&evdev->client_lock); } +static void evdev_reclaim_client(struct rcu_head *rp) +{ + struct evdev_client *client = container_of(rp, struct evdev_client, rcu); + unsigned int i; + for (i = 0; i < EV_CNT; ++i) + bitmap_free(client->evmasks[i]); + kvfree(client); +} + static void evdev_detach_client(struct evdev *evdev, struct evdev_client *client) { spin_lock(&evdev->client_lock); list_del_rcu(&client->node); spin_unlock(&evdev->client_lock); - synchronize_rcu(); + call_rcu(&client->rcu, evdev_reclaim_client); } static int evdev_open_device(struct evdev *evdev) @@ -427,7 +437,6 @@ static int evdev_release(struct inode *inode, struct file *file) { struct evdev_client *client = file->private_data; struct evdev *evdev = client->evdev; - unsigned int i; mutex_lock(&evdev->mutex); @@ -439,11 +448,6 @@ static int evdev_release(struct inode *inode, struct file *file) evdev_detach_client(evdev, client); - for (i = 0; i < EV_CNT; ++i) - bitmap_free(client->evmasks[i]); - - kvfree(client); - evdev_close_device(evdev); return 0; @@ -486,7 +490,6 @@ static int evdev_open(struct inode *inode, struct file *file) err_free_client: evdev_detach_client(evdev, client); - kvfree(client); return error; } diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig index 331b8e535e5b..80dabeebf580 100644 --- a/drivers/media/v4l2-core/Kconfig +++ b/drivers/media/v4l2-core/Kconfig @@ -40,6 +40,11 @@ config VIDEO_TUNER config V4L2_JPEG_HELPER tristate +config V4L2_LOOPBACK + tristate "V4L2 loopback device" + help + V4L2 loopback device + # Used by drivers that need v4l2-h264.ko config V4L2_H264 tristate diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile index 2177b9d63a8f..c179507cedc4 100644 --- a/drivers/media/v4l2-core/Makefile +++ b/drivers/media/v4l2-core/Makefile @@ -33,5 +33,7 @@ obj-$(CONFIG_V4L2_JPEG_HELPER) += v4l2-jpeg.o obj-$(CONFIG_V4L2_MEM2MEM_DEV) += v4l2-mem2mem.o obj-$(CONFIG_V4L2_VP9) += v4l2-vp9.o +obj-$(CONFIG_V4L2_LOOPBACK) += v4l2loopback.o + obj-$(CONFIG_VIDEO_TUNER) += tuner.o obj-$(CONFIG_VIDEO_DEV) += v4l2-dv-timings.o videodev.o diff --git a/drivers/media/v4l2-core/v4l2loopback.c b/drivers/media/v4l2-core/v4l2loopback.c new file mode 100644 index 000000000000..88ce3c048c6e --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback.c @@ -0,0 +1,3313 @@ +/* -*- c-file-style: "linux" -*- */ +/* + * v4l2loopback.c -- video4linux2 loopback driver + * + * Copyright (C) 2005-2009 Vasily Levin (vasaka@gmail.com) + * Copyright (C) 2010-2023 IOhannes m zmoelnig (zmoelnig@iem.at) + * Copyright (C) 2011 Stefan Diewald (stefan.diewald@mytum.de) + * Copyright (C) 2012 Anton Novikov (random.plant@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "v4l2loopback.h" + +#define V4L2LOOPBACK_CTL_ADD_legacy 0x4C80 +#define V4L2LOOPBACK_CTL_REMOVE_legacy 0x4C81 +#define V4L2LOOPBACK_CTL_QUERY_legacy 0x4C82 + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) +#error This module is not supported on kernels before 4.0.0. +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#define strscpy strlcpy +#endif + +#if defined(timer_setup) && defined(timer_container_of) +#define HAVE_TIMER_SETUP +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) +#define VFL_TYPE_VIDEO VFL_TYPE_GRABBER +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0) +#define timer_delete_sync del_timer_sync +#endif + +#define V4L2LOOPBACK_VERSION_CODE \ + KERNEL_VERSION(V4L2LOOPBACK_VERSION_MAJOR, V4L2LOOPBACK_VERSION_MINOR, \ + V4L2LOOPBACK_VERSION_BUGFIX) + +MODULE_DESCRIPTION("V4L2 loopback video device"); +MODULE_AUTHOR("Vasily Levin, " + "IOhannes m zmoelnig ," + "Stefan Diewald," + "Anton Novikov" + "et al."); +#ifdef SNAPSHOT_VERSION +MODULE_VERSION(__stringify(SNAPSHOT_VERSION)); +#else +MODULE_VERSION("" __stringify(V4L2LOOPBACK_VERSION_MAJOR) "." __stringify( + V4L2LOOPBACK_VERSION_MINOR) "." __stringify(V4L2LOOPBACK_VERSION_BUGFIX)); +#endif +MODULE_LICENSE("GPL"); + +/* + * helpers + */ +#define dprintk(fmt, args...) \ + do { \ + if (debug > 0) { \ + printk(KERN_INFO "v4l2-loopback[" __stringify( \ + __LINE__) "], pid(%d): " fmt, \ + task_pid_nr(current), ##args); \ + } \ + } while (0) + +#define MARK() \ + do { \ + if (debug > 1) { \ + printk(KERN_INFO "%s:%d[%s], pid(%d)\n", __FILE__, \ + __LINE__, __func__, task_pid_nr(current)); \ + } \ + } while (0) + +#define dprintkrw(fmt, args...) \ + do { \ + if (debug > 2) { \ + printk(KERN_INFO "v4l2-loopback[" __stringify( \ + __LINE__) "], pid(%d): " fmt, \ + task_pid_nr(current), ##args); \ + } \ + } while (0) + +static inline void v4l2l_get_timestamp(struct v4l2_buffer *b) +{ + struct timespec64 ts; + ktime_get_ts64(&ts); + + b->timestamp.tv_sec = ts.tv_sec; + b->timestamp.tv_usec = (ts.tv_nsec / NSEC_PER_USEC); + b->flags |= V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + b->flags &= ~V4L2_BUF_FLAG_TIMESTAMP_COPY; +} + +#if BITS_PER_LONG == 32 +#include /* do_div() for 64bit division */ +static inline int v4l2l_mod64(const s64 A, const u32 B) +{ + u64 a = (u64)A; + u32 b = B; + + if (A > 0) + return do_div(a, b); + a = -A; + return -do_div(a, b); +} +#else +static inline int v4l2l_mod64(const s64 A, const u32 B) +{ + return A % B; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) +typedef unsigned __poll_t; +#endif + +/* module constants + * can be overridden during he build process using something like + * make KCPPFLAGS="-DMAX_DEVICES=100" + */ + +/* maximum number of v4l2loopback devices that can be created */ +#ifndef MAX_DEVICES +#define MAX_DEVICES 8 +#endif + +/* whether the default is to announce capabilities exclusively or not */ +#ifndef V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +#define V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS 0 +#endif + +/* when a producer is considered to have gone stale */ +#ifndef MAX_TIMEOUT +#define MAX_TIMEOUT (100 * 1000) /* in msecs */ +#endif + +/* max buffers that can be mapped, actually they + * are all mapped to max_buffers buffers */ +#ifndef MAX_BUFFERS +#define MAX_BUFFERS 32 +#endif + +/* module parameters */ +static int debug = 0; +module_param(debug, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(debug, "debugging level (higher values == more verbose)"); + +#define V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 2 +static int max_buffers = V4L2LOOPBACK_DEFAULT_MAX_BUFFERS; +module_param(max_buffers, int, S_IRUGO); +MODULE_PARM_DESC(max_buffers, + "how many buffers should be allocated [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_MAX_BUFFERS) "]"); + +/* how many times a device can be opened + * the per-module default value can be overridden on a per-device basis using + * the /sys/devices interface + * + * note that max_openers should be at least 2 in order to get a working system: + * one opener for the producer and one opener for the consumer + * however, we leave that to the user + */ +#define V4L2LOOPBACK_DEFAULT_MAX_OPENERS 10 +static int max_openers = V4L2LOOPBACK_DEFAULT_MAX_OPENERS; +module_param(max_openers, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC( + max_openers, + "how many users can open the loopback device [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_MAX_OPENERS) "]"); + +static int devices = -1; +module_param(devices, int, 0); +MODULE_PARM_DESC(devices, "how many devices should be created"); + +static int video_nr[MAX_DEVICES] = { [0 ...(MAX_DEVICES - 1)] = -1 }; +module_param_array(video_nr, int, NULL, 0444); +MODULE_PARM_DESC(video_nr, + "video device numbers (-1=auto, 0=/dev/video0, etc.)"); + +static char *card_label[MAX_DEVICES]; +module_param_array(card_label, charp, NULL, 0000); +MODULE_PARM_DESC(card_label, "card labels for each device"); + +static bool exclusive_caps[MAX_DEVICES] = { + [0 ...(MAX_DEVICES - 1)] = V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +}; +module_param_array(exclusive_caps, bool, NULL, 0444); +/* FIXXME: wording */ +MODULE_PARM_DESC( + exclusive_caps, + "whether to announce OUTPUT/CAPTURE capabilities exclusively or not [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS) "]"); + +/* format specifications */ +#define V4L2LOOPBACK_SIZE_MIN_WIDTH 2 +#define V4L2LOOPBACK_SIZE_MIN_HEIGHT 1 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH 8192 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT 8192 + +#define V4L2LOOPBACK_SIZE_DEFAULT_WIDTH 640 +#define V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT 480 + +static int max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; +module_param(max_width, int, S_IRUGO); +MODULE_PARM_DESC(max_width, + "maximum allowed frame width [DEFAULT: " __stringify( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH) "]"); +static int max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; +module_param(max_height, int, S_IRUGO); +MODULE_PARM_DESC(max_height, + "maximum allowed frame height [DEFAULT: " __stringify( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT) "]"); + +static DEFINE_IDR(v4l2loopback_index_idr); +static DEFINE_MUTEX(v4l2loopback_ctl_mutex); + +/* frame intervals */ +#define V4L2LOOPBACK_FRAME_INTERVAL_MAX __UINT32_MAX__ +#define V4L2LOOPBACK_FPS_DEFAULT 30 +#define V4L2LOOPBACK_FPS_MAX 1000 + +/* control IDs */ +#define V4L2LOOPBACK_CID_BASE (V4L2_CID_USER_BASE | 0xf000) +#define CID_KEEP_FORMAT (V4L2LOOPBACK_CID_BASE + 0) +#define CID_SUSTAIN_FRAMERATE (V4L2LOOPBACK_CID_BASE + 1) +#define CID_TIMEOUT (V4L2LOOPBACK_CID_BASE + 2) +#define CID_TIMEOUT_IMAGE_IO (V4L2LOOPBACK_CID_BASE + 3) + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl); +static const struct v4l2_ctrl_ops v4l2loopback_ctrl_ops = { + .s_ctrl = v4l2loopback_s_ctrl, +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_keepformat = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_KEEP_FORMAT, + .name = "keep_format", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_sustainframerate = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_SUSTAIN_FRAMERATE, + .name = "sustain_framerate", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeout = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT, + .name = "timeout", + .type = V4L2_CTRL_TYPE_INTEGER, + .min = 0, + .max = MAX_TIMEOUT, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeoutimageio = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT_IMAGE_IO, + .name = "timeout_image_io", + .type = V4L2_CTRL_TYPE_BUTTON, + .min = 0, + .max = 0, + .step = 0, + .def = 0, + // clang-format on +}; + +/* module structures */ +struct v4l2loopback_private { + int device_nr; +}; + +/* TODO(vasaka) use typenames which are common to kernel, but first find out if + * it is needed */ +/* struct keeping state and settings of loopback device */ + +struct v4l2l_buffer { + struct v4l2_buffer buffer; + struct list_head list_head; + atomic_t use_count; +}; + +struct v4l2_loopback_device { + struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct video_device *vdev; + + /* loopback device-specific parameters */ + char card_label[32]; + bool announce_all_caps; /* announce both OUTPUT and CAPTURE capabilities + * when true; else announce OUTPUT when no + * writer is streaming, otherwise CAPTURE. */ + int max_openers; /* how many times can this device be opened */ + int min_width, max_width; + int min_height, max_height; + + /* pixel and stream format */ + struct v4l2_pix_format pix_format; + bool pix_format_has_valid_sizeimage; + struct v4l2_captureparm capture_param; + unsigned long frame_jiffies; + + /* ctrls */ + int keep_format; /* CID_KEEP_FORMAT; lock the format, do not free + * on close(), and when `!announce_all_caps` do NOT + * fall back to OUTPUT when no writers attached (clear + * `keep_format` to attach a new writer) */ + int sustain_framerate; /* CID_SUSTAIN_FRAMERATE; duplicate frames to maintain + (close to) nominal framerate */ + unsigned long timeout_jiffies; /* CID_TIMEOUT; 0 means disabled */ + int timeout_image_io; /* CID_TIMEOUT_IMAGE_IO; next opener will + * queue/dequeue the timeout image buffer */ + + /* buffers for OUTPUT and CAPTURE */ + u8 *image; /* pointer to actual buffers data */ + unsigned long image_size; /* number of bytes alloc'd for all buffers */ + struct v4l2l_buffer buffers[MAX_BUFFERS]; /* inner driver buffers */ + u32 buffer_count; /* should not be big, 4 is a good choice */ + u32 buffer_size; /* number of bytes alloc'd per buffer */ + u32 used_buffer_count; /* number of buffers allocated to openers */ + struct list_head outbufs_list; /* FIFO queue for OUTPUT buffers */ + u32 bufpos2index[MAX_BUFFERS]; /* mapping of `(position % used_buffers)` + * to `buffers[index]` */ + s64 write_position; /* sequence number of last 'displayed' buffer plus + * one */ + + /* synchronization between openers */ + atomic_t open_count; + struct mutex image_mutex; /* mutex for allocating image(s) and + * exchanging format tokens */ + spinlock_t lock; /* lock for the timeout and framerate timers */ + spinlock_t list_lock; /* lock for the OUTPUT buffer queue */ + wait_queue_head_t read_event; + u32 format_tokens; /* tokens to 'set format' for OUTPUT, CAPTURE, or + * timeout buffers */ + u32 stream_tokens; /* tokens to 'start' OUTPUT, CAPTURE, or timeout + * stream */ + + /* sustain framerate */ + struct timer_list sustain_timer; + unsigned int reread_count; + + /* timeout */ + u8 *timeout_image; /* copied to outgoing buffers when timeout passes */ + struct v4l2l_buffer timeout_buffer; + u32 timeout_buffer_size; /* number bytes alloc'd for timeout buffer */ + struct timer_list timeout_timer; + int timeout_happened; +}; + +enum v4l2l_io_method { + V4L2L_IO_NONE = 0, + V4L2L_IO_MMAP = 1, + V4L2L_IO_FILE = 2, + V4L2L_IO_TIMEOUT = 3, +}; + +/* struct keeping state and type of opener */ +struct v4l2_loopback_opener { + u32 format_token; /* token (if any) for type used in call to S_FMT or + * REQBUFS */ + u32 stream_token; /* token (if any) for type used in call to STREAMON */ + u32 buffer_count; /* number of buffers (if any) that opener acquired via + * REQBUFS */ + s64 read_position; /* sequence number of the next 'captured' frame */ + unsigned int reread_count; + enum v4l2l_io_method io_method; + + struct v4l2_fh fh; +}; + +#define fh_to_opener(ptr) container_of((ptr), struct v4l2_loopback_opener, fh) + +/* this is heavily inspired by the bttv driver found in the linux kernel */ +struct v4l2l_format { + char *name; + int fourcc; /* video4linux 2 */ + int depth; /* bit/pixel */ + int flags; +}; +/* set the v4l2l_format.flags to PLANAR for non-packed formats */ +#define FORMAT_FLAGS_PLANAR 0x01 +#define FORMAT_FLAGS_COMPRESSED 0x02 + +#include "v4l2loopback_formats.h" + +#ifndef V4L2_TYPE_IS_CAPTURE +#define V4L2_TYPE_IS_CAPTURE(type) \ + ((type) == V4L2_BUF_TYPE_VIDEO_CAPTURE || \ + (type) == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) +#endif /* V4L2_TYPE_IS_CAPTURE */ +#ifndef V4L2_TYPE_IS_OUTPUT +#define V4L2_TYPE_IS_OUTPUT(type) \ + ((type) == V4L2_BUF_TYPE_VIDEO_OUTPUT || \ + (type) == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) +#endif /* V4L2_TYPE_IS_OUTPUT */ + +/* token values for privilege to set format or start/stop stream */ +#define V4L2L_TOKEN_CAPTURE 0x01 +#define V4L2L_TOKEN_OUTPUT 0x02 +#define V4L2L_TOKEN_TIMEOUT 0x04 +#define V4L2L_TOKEN_MASK \ + (V4L2L_TOKEN_CAPTURE | V4L2L_TOKEN_OUTPUT | V4L2L_TOKEN_TIMEOUT) + +/* helpers for token exchange and token status */ +#define token_from_type(type) \ + (V4L2_TYPE_IS_CAPTURE(type) ? V4L2L_TOKEN_CAPTURE : V4L2L_TOKEN_OUTPUT) +#define acquire_token(dev, opener, label, token) \ + do { \ + (opener)->label##_token = token; \ + (dev)->label##_tokens &= ~token; \ + } while (0) +#define release_token(dev, opener, label) \ + do { \ + (dev)->label##_tokens |= (opener)->label##_token; \ + (opener)->label##_token = 0; \ + } while (0) +#define has_output_token(token) (token & V4L2L_TOKEN_OUTPUT) +#define has_capture_token(token) (token & V4L2L_TOKEN_CAPTURE) +#define has_no_owners(dev) ((~((dev)->format_tokens) & V4L2L_TOKEN_MASK) == 0) +#define has_other_owners(opener, dev) \ + (~((dev)->format_tokens ^ (opener)->format_token) & V4L2L_TOKEN_MASK) +#define need_timeout_buffer(dev, token) \ + ((dev)->timeout_jiffies > 0 || (token) & V4L2L_TOKEN_TIMEOUT) + +static const unsigned int FORMATS = ARRAY_SIZE(formats); + +static char *fourcc2str(unsigned int fourcc, char buf[5]) +{ + buf[0] = (fourcc >> 0) & 0xFF; + buf[1] = (fourcc >> 8) & 0xFF; + buf[2] = (fourcc >> 16) & 0xFF; + buf[3] = (fourcc >> 24) & 0xFF; + buf[4] = 0; + + return buf; +} + +static const struct v4l2l_format *format_by_fourcc(int fourcc) +{ + unsigned int i; + char buf[5]; + + for (i = 0; i < FORMATS; i++) { + if (formats[i].fourcc == fourcc) + return formats + i; + } + + dprintk("unsupported format '%4s'\n", fourcc2str(fourcc, buf)); + return NULL; +} + +static void pix_format_set_size(struct v4l2_pix_format *f, + const struct v4l2l_format *fmt, + unsigned int width, unsigned int height) +{ + f->width = width; + f->height = height; + + if (fmt->flags & FORMAT_FLAGS_PLANAR) { + f->bytesperline = width; /* Y plane */ + f->sizeimage = (width * height * fmt->depth) >> 3; + } else if (fmt->flags & FORMAT_FLAGS_COMPRESSED) { + /* doesn't make sense for compressed formats */ + f->bytesperline = 0; + f->sizeimage = (width * height * fmt->depth) >> 3; + } else { + f->bytesperline = (width * fmt->depth) >> 3; + f->sizeimage = height * f->bytesperline; + } +} + +static int v4l2l_fill_format(struct v4l2_format *fmt, const u32 minwidth, + const u32 maxwidth, const u32 minheight, + const u32 maxheight) +{ + u32 width = fmt->fmt.pix.width, height = fmt->fmt.pix.height; + u32 pixelformat = fmt->fmt.pix.pixelformat; + struct v4l2_format fmt0 = *fmt; + u32 bytesperline = 0, sizeimage = 0; + + if (!width) + width = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + if (!height) + height = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + width = clamp_val(width, minwidth, maxwidth); + height = clamp_val(height, minheight, maxheight); + + /* sets: width,height,pixelformat,bytesperline,sizeimage */ + if (!(V4L2_TYPE_IS_MULTIPLANAR(fmt0.type))) { + fmt0.fmt.pix.bytesperline = 0; + fmt0.fmt.pix.sizeimage = 0; + } + + if (0) { + ; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0) + } else if (!v4l2_fill_pixfmt(&fmt0.fmt.pix, pixelformat, width, + height)) { + ; + } else if (!v4l2_fill_pixfmt_mp(&fmt0.fmt.pix_mp, pixelformat, width, + height)) { + ; +#endif + } else { + const struct v4l2l_format *format = + format_by_fourcc(pixelformat); + if (!format) + return -EINVAL; + pix_format_set_size(&fmt0.fmt.pix, format, width, height); + fmt0.fmt.pix.pixelformat = format->fourcc; + } + + if (V4L2_TYPE_IS_MULTIPLANAR(fmt0.type)) { + *fmt = fmt0; + + if ((fmt->fmt.pix_mp.colorspace == V4L2_COLORSPACE_DEFAULT) || + (fmt->fmt.pix_mp.colorspace > V4L2_COLORSPACE_DCI_P3)) + fmt->fmt.pix_mp.colorspace = V4L2_COLORSPACE_SRGB; + if (V4L2_FIELD_ANY == fmt->fmt.pix_mp.field) + fmt->fmt.pix_mp.field = V4L2_FIELD_NONE; + } else { + bytesperline = fmt->fmt.pix.bytesperline; + sizeimage = fmt->fmt.pix.sizeimage; + + *fmt = fmt0; + + if (!fmt->fmt.pix.bytesperline) + fmt->fmt.pix.bytesperline = bytesperline; + if (!fmt->fmt.pix.sizeimage) + fmt->fmt.pix.sizeimage = sizeimage; + + if ((fmt->fmt.pix.colorspace == V4L2_COLORSPACE_DEFAULT) || + (fmt->fmt.pix.colorspace > V4L2_COLORSPACE_DCI_P3)) + fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + if (V4L2_FIELD_ANY == fmt->fmt.pix.field) + fmt->fmt.pix.field = V4L2_FIELD_NONE; + } + + return 0; +} + +/* Checks if v4l2l_fill_format() has set a valid, fixed sizeimage val. */ +static bool v4l2l_pix_format_has_valid_sizeimage(struct v4l2_format *fmt) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0) + const struct v4l2_format_info *info; + + info = v4l2_format_info(fmt->fmt.pix.pixelformat); + if (info && info->mem_planes == 1) + return true; +#endif + + return false; +} + +static int pix_format_eq(const struct v4l2_pix_format *ref, + const struct v4l2_pix_format *tgt, int strict) +{ + /* check if the two formats are equivalent. + * ANY fields are handled gracefully + */ +#define _pix_format_eq0(x) \ + if (ref->x != tgt->x) \ + result = 0 +#define _pix_format_eq1(x, def) \ + do { \ + if ((def != tgt->x) && (ref->x != tgt->x)) { \ + printk(KERN_INFO #x " failed"); \ + result = 0; \ + } \ + } while (0) + int result = 1; + _pix_format_eq0(width); + _pix_format_eq0(height); + _pix_format_eq0(pixelformat); + if (!strict) + return result; + _pix_format_eq1(field, V4L2_FIELD_ANY); + _pix_format_eq0(bytesperline); + _pix_format_eq0(sizeimage); + _pix_format_eq1(colorspace, V4L2_COLORSPACE_DEFAULT); + return result; +} + +static void set_timeperframe(struct v4l2_loopback_device *dev, + struct v4l2_fract *tpf) +{ + if (!tpf->denominator && !tpf->numerator) { + tpf->numerator = 1; + tpf->denominator = V4L2LOOPBACK_FPS_DEFAULT; + } else if (tpf->numerator > + V4L2LOOPBACK_FRAME_INTERVAL_MAX * tpf->denominator) { + /* divide-by-zero or greater than maximum interval => min FPS */ + tpf->numerator = V4L2LOOPBACK_FRAME_INTERVAL_MAX; + tpf->denominator = 1; + } else if (tpf->numerator * V4L2LOOPBACK_FPS_MAX < tpf->denominator) { + /* zero or lower than minimum interval => max FPS */ + tpf->numerator = 1; + tpf->denominator = V4L2LOOPBACK_FPS_MAX; + } + + dev->capture_param.timeperframe = *tpf; + dev->frame_jiffies = + max(1UL, (msecs_to_jiffies(1000) * tpf->numerator) / + tpf->denominator); +} + +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd); + +/* device attributes */ +/* available via sysfs: /sys/devices/virtual/video4linux/video* */ + +static ssize_t attr_show_format(struct device *cd, + struct device_attribute *attr, char *buf) +{ + /* gets the current format as "FOURCC:WxH@f/s", e.g. "YUYV:320x240@1000/30" */ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + const struct v4l2_fract *tpf; + char buf4cc[5], buf_fps[32]; + + if (!dev || (has_no_owners(dev) && !dev->keep_format)) + return 0; + tpf = &dev->capture_param.timeperframe; + + fourcc2str(dev->pix_format.pixelformat, buf4cc); + if (tpf->numerator == 1) + snprintf(buf_fps, sizeof(buf_fps), "%u", tpf->denominator); + else + snprintf(buf_fps, sizeof(buf_fps), "%u/%u", tpf->denominator, + tpf->numerator); + return sprintf(buf, "%4s:%ux%u@%s\n", buf4cc, dev->pix_format.width, + dev->pix_format.height, buf_fps); +} + +static ssize_t attr_store_format(struct device *cd, + struct device_attribute *attr, const char *buf, + size_t len) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + int fps_num = 0, fps_den = 1; + + if (!dev) + return -ENODEV; + + /* only fps changing is supported */ + if (sscanf(buf, "@%u/%u", &fps_num, &fps_den) > 0) { + struct v4l2_fract f = { .numerator = fps_den, + .denominator = fps_num }; + set_timeperframe(dev, &f); + return len; + } + return -EINVAL; +} + +static DEVICE_ATTR(format, S_IRUGO | S_IWUSR, attr_show_format, + attr_store_format); + +static ssize_t attr_show_buffers(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + return sprintf(buf, "%u\n", dev->used_buffer_count); +} + +static DEVICE_ATTR(buffers, S_IRUGO, attr_show_buffers, NULL); + +static ssize_t attr_show_maxopeners(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + return sprintf(buf, "%d\n", dev->max_openers); +} + +static ssize_t attr_store_maxopeners(struct device *cd, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct v4l2_loopback_device *dev = NULL; + unsigned long curr = 0; + + if (kstrtoul(buf, 0, &curr)) + return -EINVAL; + + dev = v4l2loopback_cd2dev(cd); + if (!dev) + return -ENODEV; + + if (dev->max_openers == curr) + return len; + + if (curr > __INT_MAX__ || dev->open_count.counter > curr) { + /* request to limit to less openers as are currently attached to us */ + return -EINVAL; + } + + dev->max_openers = (int)curr; + + return len; +} + +static DEVICE_ATTR(max_openers, S_IRUGO | S_IWUSR, attr_show_maxopeners, + attr_store_maxopeners); + +static ssize_t attr_show_state(struct device *cd, struct device_attribute *attr, + char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + if (!has_output_token(dev->stream_tokens) || dev->keep_format) { + return sprintf(buf, "capture\n"); + } else + return sprintf(buf, "output\n"); + + return -EAGAIN; +} + +static DEVICE_ATTR(state, S_IRUGO, attr_show_state, NULL); + +static void v4l2loopback_remove_sysfs(struct video_device *vdev) +{ +#define V4L2_SYSFS_DESTROY(x) device_remove_file(&vdev->dev, &dev_attr_##x) + + if (vdev) { + V4L2_SYSFS_DESTROY(format); + V4L2_SYSFS_DESTROY(buffers); + V4L2_SYSFS_DESTROY(max_openers); + V4L2_SYSFS_DESTROY(state); + /* ... */ + } +} + +static void v4l2loopback_create_sysfs(struct video_device *vdev) +{ + int res = 0; + +#define V4L2_SYSFS_CREATE(x) \ + res = device_create_file(&vdev->dev, &dev_attr_##x); \ + if (res < 0) \ + break + if (!vdev) + return; + do { + V4L2_SYSFS_CREATE(format); + V4L2_SYSFS_CREATE(buffers); + V4L2_SYSFS_CREATE(max_openers); + V4L2_SYSFS_CREATE(state); + /* ... */ + } while (0); + + if (res >= 0) + return; + dev_err(&vdev->dev, "%s error: %d\n", __func__, res); +} + +/* Event APIs */ + +#define V4L2LOOPBACK_EVENT_BASE (V4L2_EVENT_PRIVATE_START) +#define V4L2LOOPBACK_EVENT_OFFSET 0x08E00000 +#define V4L2_EVENT_PRI_CLIENT_USAGE \ + (V4L2LOOPBACK_EVENT_BASE + V4L2LOOPBACK_EVENT_OFFSET + 1) + +struct v4l2_event_client_usage { + __u32 count; +}; + +/* global module data */ +/* find a device based on it's device-number (e.g. '3' for /dev/video3) */ +struct v4l2loopback_lookup_cb_data { + int device_nr; + struct v4l2_loopback_device *device; +}; +static int v4l2loopback_lookup_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *device = ptr; + struct v4l2loopback_lookup_cb_data *cbdata = data; + if (cbdata && device && device->vdev) { + if (device->vdev->num == cbdata->device_nr) { + cbdata->device = device; + cbdata->device_nr = id; + return 1; + } + } + return 0; +} +static int v4l2loopback_lookup(int device_nr, + struct v4l2_loopback_device **device) +{ + struct v4l2loopback_lookup_cb_data data = { + .device_nr = device_nr, + .device = NULL, + }; + int err = idr_for_each(&v4l2loopback_index_idr, &v4l2loopback_lookup_cb, + &data); + if (1 == err) { + if (device) + *device = data.device; + return data.device_nr; + } + return -ENODEV; +} +#define v4l2loopback_get_vdev_nr(vdev) \ + ((struct v4l2loopback_private *)video_get_drvdata(vdev))->device_nr +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd) +{ + struct video_device *loopdev = to_video_device(cd); + int device_nr = v4l2loopback_get_vdev_nr(loopdev); + + return idr_find(&v4l2loopback_index_idr, device_nr); +} + +static struct v4l2_loopback_device *v4l2loopback_getdevice(struct file *f) +{ + struct v4l2loopback_private *ptr = video_drvdata(f); + int nr = ptr->device_nr; + + return idr_find(&v4l2loopback_index_idr, nr); +} + +/* forward declarations */ +static void client_usage_queue_event(struct video_device *vdev); +static bool any_buffers_mapped(struct v4l2_loopback_device *dev); +static int allocate_buffers(struct v4l2_loopback_device *dev, + struct v4l2_pix_format *pix_format); +static void init_buffers(struct v4l2_loopback_device *dev, u32 bytes_used, + u32 buffer_size); +static void free_buffers(struct v4l2_loopback_device *dev); +static int allocate_timeout_buffer(struct v4l2_loopback_device *dev); +static void free_timeout_buffer(struct v4l2_loopback_device *dev); +static void check_timers(struct v4l2_loopback_device *dev); +static const struct v4l2_file_operations v4l2_loopback_fops; +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops; + +/* V4L2 ioctl caps and params calls */ +/* returns device capabilities + * called on VIDIOC_QUERYCAP + */ +static int vidioc_querycap(struct file *file, void *fh, + struct v4l2_capability *cap) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + int device_nr = v4l2loopback_get_vdev_nr(dev->vdev); + __u32 capabilities = V4L2_CAP_STREAMING | V4L2_CAP_READWRITE; + + strscpy(cap->driver, "v4l2 loopback", sizeof(cap->driver)); + snprintf(cap->card, sizeof(cap->card), "%s", dev->card_label); + snprintf(cap->bus_info, sizeof(cap->bus_info), + "platform:v4l2loopback-%03d", device_nr); + + if (dev->announce_all_caps) { + capabilities |= V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OUTPUT; + } else { + if (opener->io_method == V4L2L_IO_TIMEOUT || + (has_output_token(dev->stream_tokens) && + !dev->keep_format)) { + capabilities |= V4L2_CAP_VIDEO_OUTPUT; + } else + capabilities |= V4L2_CAP_VIDEO_CAPTURE; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + dev->vdev->device_caps = +#endif /* >=linux-4.7.0 */ + cap->device_caps = cap->capabilities = capabilities; + + cap->capabilities |= V4L2_CAP_DEVICE_CAPS; + + memset(cap->reserved, 0, sizeof(cap->reserved)); + return 0; +} + +static int vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + if (dev->keep_format || has_other_owners(opener, dev)) { + /* only current frame size supported */ + if (argp->pixel_format != dev->pix_format.pixelformat) + return -EINVAL; + + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->pix_format.width; + argp->discrete.height = dev->pix_format.height; + } else { + /* return continuous sizes if pixel format is supported */ + if (NULL == format_by_fourcc(argp->pixel_format)) + return -EINVAL; + + if (dev->min_width == dev->max_width && + dev->min_height == dev->max_height) { + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->min_width; + argp->discrete.height = dev->min_height; + } else { + argp->type = V4L2_FRMSIZE_TYPE_CONTINUOUS; + + argp->stepwise.min_width = dev->min_width; + argp->stepwise.min_height = dev->min_height; + + argp->stepwise.max_width = dev->max_width; + argp->stepwise.max_height = dev->max_height; + + argp->stepwise.step_width = 1; + argp->stepwise.step_height = 1; + } + } + return 0; +} + +/* Test if the device is currently 'capable' of the buffer (stream) type when + * the `exclusive_caps` parameter is set. `keep_format` should lock the format + * and prevent free of buffers */ +static int check_buffer_capability(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener, + enum v4l2_buf_type type) +{ + /* short-circuit for (non-compliant) timeout image mode */ + if (opener->io_method == V4L2L_IO_TIMEOUT) + return 0; + if (dev->announce_all_caps) + return (type == V4L2_BUF_TYPE_VIDEO_CAPTURE || + type == V4L2_BUF_TYPE_VIDEO_OUTPUT) ? + 0 : + -EINVAL; + /* CAPTURE if opener has a capture format or a writer is streaming; + * else OUTPUT. */ + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (!(has_capture_token(opener->format_token) || + !has_output_token(dev->stream_tokens))) + return -EINVAL; + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (!(has_output_token(opener->format_token) || + has_output_token(dev->stream_tokens))) + return -EINVAL; + break; + default: + return -EINVAL; + } + return 0; +} +/* returns frameinterval (fps) for the set resolution + * called on VIDIOC_ENUM_FRAMEINTERVALS + */ +static int vidioc_enum_frameintervals(struct file *file, void *fh, + struct v4l2_frmivalenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + if (dev->keep_format || has_other_owners(opener, dev)) { + /* keep_format also locks the frame rate */ + if (argp->width != dev->pix_format.width || + argp->height != dev->pix_format.height || + argp->pixel_format != dev->pix_format.pixelformat) + return -EINVAL; + + argp->type = V4L2_FRMIVAL_TYPE_DISCRETE; + argp->discrete = dev->capture_param.timeperframe; + } else { + if (argp->width < dev->min_width || + argp->width > dev->max_width || + argp->height < dev->min_height || + argp->height > dev->max_height || + !format_by_fourcc(argp->pixel_format)) + return -EINVAL; + + argp->type = V4L2_FRMIVAL_TYPE_CONTINUOUS; + argp->stepwise.min.numerator = 1; + argp->stepwise.min.denominator = V4L2LOOPBACK_FPS_MAX; + argp->stepwise.max.numerator = V4L2LOOPBACK_FRAME_INTERVAL_MAX; + argp->stepwise.max.denominator = 1; + argp->stepwise.step.numerator = 1; + argp->stepwise.step.denominator = 1; + } + + return 0; +} + +/* Enumerate device formats + * Returns: + * - EINVAL the index is out of bounds; or if non-zero when format is fixed + * - EFAULT unexpected null pointer */ +static int vidioc_enum_fmt_vid(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + int fixed = dev->keep_format || has_other_owners(opener, dev); + const struct v4l2l_format *fmt; + + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + + if (!(f->index < FORMATS)) + return -EINVAL; + /* TODO: Support 6.14 V4L2_FMTDESC_FLAG_ENUM_ALL */ + if (fixed && f->index) + return -EINVAL; + + fmt = fixed ? format_by_fourcc(dev->pix_format.pixelformat) : + &formats[f->index]; + if (!fmt) + return -EFAULT; + + f->flags = 0; + if (fmt->flags & FORMAT_FLAGS_COMPRESSED) + f->flags |= V4L2_FMT_FLAG_COMPRESSED; + snprintf(f->description, sizeof(f->description), fmt->name); + f->pixelformat = fmt->fourcc; + return 0; +} + +/* Tests (or tries) the format. + * Returns: + * - EINVAL if the buffer type or format is not supported + */ +static int vidioc_try_fmt_vid(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + if (v4l2l_fill_format(f, dev->min_width, dev->max_width, + dev->min_height, dev->max_height) != 0) + return -EINVAL; + if (dev->keep_format || has_other_owners(opener, dev)) + /* use existing format - including colorspace info */ + f->fmt.pix = dev->pix_format; + + return 0; +} + +/* Sets new format. Fills 'f' argument with the requested or existing format. + * Side-effect: buffers are allocated for the (returned) format. + * Returns: + * - EINVAL if the type is not supported + * - EBUSY if buffers are already allocated + * TODO: (vasaka) set subregions of input + */ +static int vidioc_s_fmt_vid(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = opener->io_method == V4L2L_IO_TIMEOUT ? + V4L2L_TOKEN_TIMEOUT : + token_from_type(f->type); + int changed, result; + char buf[5]; + + result = vidioc_try_fmt_vid(file, fh, f); + if (result < 0) + return result; + + if (opener->buffer_count > 0) + /* must free buffers before format can be set */ + return -EBUSY; + + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + + if (opener->format_token) + release_token(dev, opener, format); + if (!(dev->format_tokens & token)) { + result = -EBUSY; + goto exit_s_fmt_unlock; + } + + dprintk("S_FMT[%s] %4s:%ux%u size=%u\n", + V4L2_TYPE_IS_CAPTURE(f->type) ? "CAPTURE" : "OUTPUT", + fourcc2str(f->fmt.pix.pixelformat, buf), f->fmt.pix.width, + f->fmt.pix.height, f->fmt.pix.sizeimage); + changed = !pix_format_eq(&dev->pix_format, &f->fmt.pix, 0); + if (changed || has_no_owners(dev)) { + result = allocate_buffers(dev, &f->fmt.pix); + if (result < 0) + goto exit_s_fmt_unlock; + } + if ((dev->timeout_image && changed) || + (!dev->timeout_image && need_timeout_buffer(dev, token))) { + result = allocate_timeout_buffer(dev); + if (result < 0) + goto exit_s_fmt_free; + } + if (changed) { + dev->pix_format = f->fmt.pix; + dev->pix_format_has_valid_sizeimage = + v4l2l_pix_format_has_valid_sizeimage(f); + } + acquire_token(dev, opener, format, token); + if (opener->io_method == V4L2L_IO_TIMEOUT) + dev->timeout_image_io = 0; + goto exit_s_fmt_unlock; +exit_s_fmt_free: + free_buffers(dev); +exit_s_fmt_unlock: + mutex_unlock(&dev->image_mutex); + return result; +} + +/* ------------------ CAPTURE ----------------------- */ +/* ioctl for VIDIOC_ENUM_FMT, _G_FMT, _S_FMT, and _TRY_FMT when buffer type + * is V4L2_BUF_TYPE_VIDEO_CAPTURE */ + +static int vidioc_enum_fmt_cap(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt_vid(file, fh, f); +} + +static int vidioc_g_fmt_cap(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + f->fmt.pix = dev->pix_format; + return 0; +} + +static int vidioc_try_fmt_cap(struct file *file, void *fh, + struct v4l2_format *f) +{ + return vidioc_try_fmt_vid(file, fh, f); +} + +static int vidioc_s_fmt_cap(struct file *file, void *fh, struct v4l2_format *f) +{ + return vidioc_s_fmt_vid(file, fh, f); +} + +/* ------------------ OUTPUT ----------------------- */ +/* ioctl for VIDIOC_ENUM_FMT, _G_FMT, _S_FMT, and _TRY_FMT when buffer type + * is V4L2_BUF_TYPE_VIDEO_OUTPUT */ + +static int vidioc_enum_fmt_out(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt_vid(file, fh, f); +} + +static int vidioc_g_fmt_out(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + /* + * LATER: this should return the currently valid format + * gstreamer doesn't like it, if this returns -EINVAL, as it + * then concludes that there is _no_ valid format + * CHECK whether this assumption is wrong, + * or whether we have to always provide a valid format + */ + f->fmt.pix = dev->pix_format; + return 0; +} + +static int vidioc_try_fmt_out(struct file *file, void *fh, + struct v4l2_format *f) +{ + return vidioc_try_fmt_vid(file, fh, f); +} + +static int vidioc_s_fmt_out(struct file *file, void *fh, struct v4l2_format *f) +{ + return vidioc_s_fmt_vid(file, fh, f); +} + +// #define V4L2L_OVERLAY +#ifdef V4L2L_OVERLAY +/* ------------------ OVERLAY ----------------------- */ +/* currently unsupported */ +/* GSTreamer's v4l2sink is buggy, as it requires the overlay to work + * while it should only require it, if overlay is requested + * once the gstreamer element is fixed, remove the overlay dummies + */ +#warning OVERLAY dummies +static int vidioc_g_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} + +static int vidioc_s_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} +#endif /* V4L2L_OVERLAY */ + +/* ------------------ PARAMs ----------------------- */ + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_G_PARM + */ +static int vidioc_g_parm(struct file *file, void *fh, + struct v4l2_streamparm *parm) +{ + /* do not care about type of opener, hope these enums would always be + * compatible */ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, parm->type) < 0) + return -EINVAL; + parm->parm.capture = dev->capture_param; + return 0; +} + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_S_PARM + */ +static int vidioc_s_parm(struct file *file, void *fh, + struct v4l2_streamparm *parm) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + dprintk("S_PARM(frame-time=%u/%u)\n", + parm->parm.capture.timeperframe.numerator, + parm->parm.capture.timeperframe.denominator); + if (check_buffer_capability(dev, opener, parm->type) < 0) + return -EINVAL; + + switch (parm->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + set_timeperframe(dev, &parm->parm.capture.timeperframe); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + set_timeperframe(dev, &parm->parm.output.timeperframe); + break; + default: + return -EINVAL; + } + + parm->parm.capture = dev->capture_param; + return 0; +} + +#ifdef V4L2LOOPBACK_WITH_STD +/* sets a tv standard, actually we do not need to handle this any special way + * added to support effecttv + * called on VIDIOC_S_STD + */ +static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id *_std) +{ + v4l2_std_id req_std = 0, supported_std = 0; + const v4l2_std_id all_std = V4L2_STD_ALL, no_std = 0; + + if (_std) { + req_std = *_std; + *_std = all_std; + } + + /* we support everything in V4L2_STD_ALL, but not more... */ + supported_std = (all_std & req_std); + if (no_std == supported_std) + return -EINVAL; + + return 0; +} + +/* gets a fake video standard + * called on VIDIOC_G_STD + */ +static int vidioc_g_std(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +/* gets a fake video standard + * called on VIDIOC_QUERYSTD + */ +static int vidioc_querystd(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +#endif /* V4L2LOOPBACK_WITH_STD */ + +static int v4l2loopback_set_ctrl(struct v4l2_loopback_device *dev, u32 id, + s64 val) +{ + int result = 0; + switch (id) { + case CID_KEEP_FORMAT: + if (val < 0 || val > 1) + return -EINVAL; + dev->keep_format = val; + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + if (!dev->keep_format) { + if (has_no_owners(dev) && !any_buffers_mapped(dev)) + free_buffers(dev); + } + mutex_unlock(&dev->image_mutex); + break; + case CID_SUSTAIN_FRAMERATE: + if (val < 0 || val > 1) + return -EINVAL; + spin_lock_bh(&dev->lock); + dev->sustain_framerate = val; + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT: + if (val < 0 || val > MAX_TIMEOUT) + return -EINVAL; + if (val > 0) { + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + /* on-the-fly allocate if device is owned; else + * allocate occurs on next S_FMT or REQBUFS */ + if (!has_no_owners(dev)) + result = allocate_timeout_buffer(dev); + mutex_unlock(&dev->image_mutex); + if (result < 0) { + /* disable timeout as buffer not alloc'd */ + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = 0; + spin_unlock_bh(&dev->lock); + return result; + } + } + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = msecs_to_jiffies(val); + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT_IMAGE_IO: + dev->timeout_image_io = 1; + break; + default: + return -EINVAL; + } + return 0; +} + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct v4l2_loopback_device *dev = container_of( + ctrl->handler, struct v4l2_loopback_device, ctrl_handler); + return v4l2loopback_set_ctrl(dev, ctrl->id, ctrl->val); +} + +/* returns set of device outputs, in our case there is only one + * called on VIDIOC_ENUMOUTPUT + */ +static int vidioc_enum_output(struct file *file, void *fh, + struct v4l2_output *outp) +{ + __u32 index = outp->index; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + if (index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(outp, 0, sizeof(*outp)); + + outp->index = index; + strscpy(outp->name, "loopback in", sizeof(outp->name)); + outp->type = V4L2_OUTPUT_TYPE_ANALOG; + outp->audioset = 0; + outp->modulator = 0; +#ifdef V4L2LOOPBACK_WITH_STD + outp->std = V4L2_STD_ALL; +#ifdef V4L2_OUT_CAP_STD + outp->capabilities |= V4L2_OUT_CAP_STD; +#endif /* V4L2_OUT_CAP_STD */ +#endif /* V4L2LOOPBACK_WITH_STD */ + + return 0; +} + +/* which output is currently active, + * called on VIDIOC_G_OUTPUT + */ +static int vidioc_g_output(struct file *file, void *fh, unsigned int *index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + if (index) + *index = 0; + return 0; +} + +/* set output, can make sense if we have more than one video src, + * called on VIDIOC_S_OUTPUT + */ +static int vidioc_s_output(struct file *file, void *fh, unsigned int index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + return index == 0 ? index : -EINVAL; +} + +/* returns set of device inputs, in our case there is only one, + * but later I may add more + * called on VIDIOC_ENUMINPUT + */ +static int vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + __u32 index = inp->index; + + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; + if (index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(inp, 0, sizeof(*inp)); + + inp->index = index; + strscpy(inp->name, "loopback", sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + inp->audioset = 0; + inp->tuner = 0; + inp->status = 0; + +#ifdef V4L2LOOPBACK_WITH_STD + inp->std = V4L2_STD_ALL; +#ifdef V4L2_IN_CAP_STD + inp->capabilities |= V4L2_IN_CAP_STD; +#endif +#endif /* V4L2LOOPBACK_WITH_STD */ + + if (has_output_token(dev->stream_tokens) && !dev->keep_format) + /* if no outputs attached; pretend device is powered off */ + inp->status |= V4L2_IN_ST_NO_SIGNAL; + + return 0; +} + +/* which input is currently active, + * called on VIDIOC_G_INPUT + */ +static int vidioc_g_input(struct file *file, void *fh, unsigned int *index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; /* NOTE: -EAGAIN might be more informative */ + if (index) + *index = 0; + return 0; +} + +/* set input, can make sense if we have more than one video src, + * called on VIDIOC_S_INPUT + */ +static int vidioc_s_input(struct file *file, void *fh, unsigned int index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (index != 0) + return -EINVAL; + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; /* NOTE: -EAGAIN might be more informative */ + return 0; +} + +/* --------------- V4L2 ioctl buffer related calls ----------------- */ + +#define is_allocated(opener, type, index) \ + (opener->format_token & (opener->io_method == V4L2L_IO_TIMEOUT ? \ + V4L2L_TOKEN_TIMEOUT : \ + token_from_type(type)) && \ + (index) < (opener)->buffer_count) +#define BUFFER_DEBUG_FMT_STR \ + "buffer#%u @ %p type=%u bytesused=%u length=%u flags=%x " \ + "field=%u timestamp= %lld.%06lldsequence=%u\n" +#define BUFFER_DEBUG_FMT_ARGS(buf) \ + (buf)->index, (buf), (buf)->type, (buf)->bytesused, (buf)->length, \ + (buf)->flags, (buf)->field, \ + (long long)(buf)->timestamp.tv_sec, \ + (long long)(buf)->timestamp.tv_usec, (buf)->sequence +/* Buffer flag helpers */ +#define unset_flags(flags) \ + do { \ + flags &= ~V4L2_BUF_FLAG_QUEUED; \ + flags &= ~V4L2_BUF_FLAG_DONE; \ + } while (0) +#define set_queued(flags) \ + do { \ + flags |= V4L2_BUF_FLAG_QUEUED; \ + flags &= ~V4L2_BUF_FLAG_DONE; \ + } while (0) +#define set_done(flags) \ + do { \ + flags &= ~V4L2_BUF_FLAG_QUEUED; \ + flags |= V4L2_BUF_FLAG_DONE; \ + } while (0) + +static bool any_buffers_mapped(struct v4l2_loopback_device *dev) +{ + u32 index; + for (index = 0; index < dev->buffer_count; ++index) + if (dev->buffers[index].buffer.flags & V4L2_BUF_FLAG_MAPPED) + return true; + return false; +} + +static void prepare_buffer_queue(struct v4l2_loopback_device *dev, int count) +{ + struct v4l2l_buffer *bufd, *n; + u32 pos; + + spin_lock_bh(&dev->list_lock); + + /* ensure sufficient number of buffers in queue */ + for (pos = 0; pos < count; ++pos) { + bufd = &dev->buffers[pos]; + if (list_empty(&bufd->list_head)) + list_add_tail(&bufd->list_head, &dev->outbufs_list); + } + if (list_empty(&dev->outbufs_list)) + goto exit_prepare_queue_unlock; + + /* remove any excess buffers */ + list_for_each_entry_safe(bufd, n, &dev->outbufs_list, list_head) { + if (bufd->buffer.index >= count) + list_del_init(&bufd->list_head); + } + + /* buffers are no longer queued; and `write_position` will correspond + * to the first item of `outbufs_list`. */ + pos = v4l2l_mod64(dev->write_position, count); + list_for_each_entry(bufd, &dev->outbufs_list, list_head) { + unset_flags(bufd->buffer.flags); + dev->bufpos2index[pos % count] = bufd->buffer.index; + ++pos; + } +exit_prepare_queue_unlock: + spin_unlock_bh(&dev->list_lock); +} + +/* forward declaration */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type); +/* negotiate buffer type + * only mmap streaming supported + * called on VIDIOC_REQBUFS + */ +static int vidioc_reqbufs(struct file *file, void *fh, + struct v4l2_requestbuffers *reqbuf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = opener->io_method == V4L2L_IO_TIMEOUT ? + V4L2L_TOKEN_TIMEOUT : + token_from_type(reqbuf->type); + u32 req_count = reqbuf->count; + int result = 0; + + dprintk("REQBUFS(memory=%u, req_count=%u) and device-bufs=%u/%u " + "[used/max]\n", + reqbuf->memory, req_count, dev->used_buffer_count, + dev->buffer_count); + + switch (reqbuf->memory) { + case V4L2_MEMORY_MMAP: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) + reqbuf->capabilities = 0; /* only guarantee MMAP support */ +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + reqbuf->flags = 0; /* no memory consistency support */ +#endif + break; + default: + return -EINVAL; + } + + if (opener->format_token & ~token) + /* different (buffer) type already assigned to descriptor by + * S_FMT or REQBUFS */ + return -EINVAL; + + MARK(); + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; /* -EINTR */ + + /* CASE queue/dequeue timeout-buffer only: */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + opener->buffer_count = req_count; + if (req_count == 0) + release_token(dev, opener, format); + goto exit_reqbufs_unlock; + } + + MARK(); + /* CASE count is zero: streamoff, free buffers, release their token */ + if (req_count == 0) { + if (dev->format_tokens & token) { + acquire_token(dev, opener, format, token); + opener->io_method = V4L2L_IO_MMAP; + } + result = vidioc_streamoff(file, fh, reqbuf->type); + opener->buffer_count = 0; + /* undocumented requirement - REQBUFS with count zero should + * ALSO release lock on logical stream */ + if (opener->format_token) + release_token(dev, opener, format); + if (has_no_owners(dev)) + dev->used_buffer_count = 0; + goto exit_reqbufs_unlock; + } + + /* CASE count non-zero: allocate buffers and acquire token for them */ + MARK(); + switch (reqbuf->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (!(dev->format_tokens & token || + opener->format_token & token)) + /* only exclusive ownership for each stream */ + result = -EBUSY; + break; + default: + result = -EINVAL; + } + if (result < 0) + goto exit_reqbufs_unlock; + + if (has_other_owners(opener, dev) && dev->used_buffer_count > 0) { + /* allow 'allocation' of existing number of buffers */ + req_count = dev->used_buffer_count; + } else if (any_buffers_mapped(dev)) { + /* do not allow re-allocation if buffers are mapped */ + result = -EBUSY; + goto exit_reqbufs_unlock; + } + + MARK(); + opener->buffer_count = 0; + + if (req_count > dev->buffer_count) + req_count = dev->buffer_count; + + if (has_no_owners(dev)) { + result = allocate_buffers(dev, &dev->pix_format); + if (result < 0) + goto exit_reqbufs_unlock; + } + if (!dev->timeout_image && need_timeout_buffer(dev, token)) { + result = allocate_timeout_buffer(dev); + if (result < 0) + goto exit_reqbufs_unlock; + } + acquire_token(dev, opener, format, token); + + MARK(); + switch (opener->io_method) { + case V4L2L_IO_TIMEOUT: + dev->timeout_image_io = 0; + opener->buffer_count = req_count; + break; + default: + opener->io_method = V4L2L_IO_MMAP; + prepare_buffer_queue(dev, req_count); + dev->used_buffer_count = opener->buffer_count = req_count; + } +exit_reqbufs_unlock: + mutex_unlock(&dev->image_mutex); + reqbuf->count = opener->buffer_count; + return result; +} + +/* returns buffer asked for; + * give app as many buffers as it wants, if it less than MAX, + * but map them in our inner buffers + * called on VIDIOC_QUERYBUF + */ +static int vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 type = buf->type; + u32 index = buf->index; + + if ((type != V4L2_BUF_TYPE_VIDEO_CAPTURE) && + (type != V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -EINVAL; + if (!is_allocated(opener, type, index)) + return -EINVAL; + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + *buf = dev->timeout_buffer.buffer; + buf->index = index; + } else + *buf = dev->buffers[index].buffer; + + buf->type = type; + + if (!(buf->flags & (V4L2_BUF_FLAG_DONE | V4L2_BUF_FLAG_QUEUED))) { + /* v4l2-compliance requires these to be zero */ + buf->sequence = 0; + buf->timestamp.tv_sec = buf->timestamp.tv_usec = 0; + } else if (V4L2_TYPE_IS_CAPTURE(type)) { + /* guess flags based on sequence values */ + if (buf->sequence >= opener->read_position) { + set_done(buf->flags); + } else if (buf->flags & V4L2_BUF_FLAG_DONE) { + set_queued(buf->flags); + } + } + dprintkrw("QUERYBUF(%s, index=%u) -> " BUFFER_DEBUG_FMT_STR, + V4L2_TYPE_IS_CAPTURE(type) ? "CAPTURE" : "OUTPUT", index, + BUFFER_DEBUG_FMT_ARGS(buf)); + return 0; +} + +static void buffer_written(struct v4l2_loopback_device *dev, + struct v4l2l_buffer *buf) +{ + timer_delete_sync(&dev->sustain_timer); + timer_delete_sync(&dev->timeout_timer); + + spin_lock_bh(&dev->list_lock); + list_move_tail(&buf->list_head, &dev->outbufs_list); + spin_unlock_bh(&dev->list_lock); + + spin_lock_bh(&dev->lock); + dev->bufpos2index[v4l2l_mod64(dev->write_position, + dev->used_buffer_count)] = + buf->buffer.index; + ++dev->write_position; + dev->reread_count = 0; + + check_timers(dev); + spin_unlock_bh(&dev->lock); +} + +/* put buffer to queue + * called on VIDIOC_QBUF + */ +static int vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + struct v4l2l_buffer *bufd; + u32 index = buf->index; + u32 type = buf->type; + + if (!is_allocated(opener, type, index)) + return -EINVAL; + bufd = &dev->buffers[index]; + + switch (buf->memory) { + case V4L2_MEMORY_MMAP: + if (!(bufd->buffer.flags & V4L2_BUF_FLAG_MAPPED)) + dprintkrw("QBUF() unmapped buffer [index=%u]\n", index); + break; + default: + return -EINVAL; + } + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + set_queued(buf->flags); + return 0; + } + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + dprintkrw("QBUF(CAPTURE, index=%u) -> " BUFFER_DEBUG_FMT_STR, + index, BUFFER_DEBUG_FMT_ARGS(buf)); + set_queued(buf->flags); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + dprintkrw("QBUF(OUTPUT, index=%u) -> " BUFFER_DEBUG_FMT_STR, + index, BUFFER_DEBUG_FMT_ARGS(buf)); + if (!(bufd->buffer.flags & V4L2_BUF_FLAG_TIMESTAMP_COPY) && + (buf->timestamp.tv_sec == 0 && + buf->timestamp.tv_usec == 0)) { + v4l2l_get_timestamp(&bufd->buffer); + } else { + bufd->buffer.timestamp = buf->timestamp; + bufd->buffer.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + bufd->buffer.flags &= + ~V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + } + if (dev->pix_format_has_valid_sizeimage) { + if (buf->bytesused >= dev->pix_format.sizeimage) { + bufd->buffer.bytesused = + dev->pix_format.sizeimage; + } else { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) + dev_warn_ratelimited( + &dev->vdev->dev, +#else + dprintkrw( +#endif + "warning queued output buffer bytesused too small %u < %u\n", + buf->bytesused, + dev->pix_format.sizeimage); + bufd->buffer.bytesused = buf->bytesused; + } + } else { + bufd->buffer.bytesused = buf->bytesused; + } + bufd->buffer.sequence = dev->write_position; + set_queued(bufd->buffer.flags); + *buf = bufd->buffer; + buffer_written(dev, bufd); + set_done(bufd->buffer.flags); + wake_up_all(&dev->read_event); + break; + default: + return -EINVAL; + } + buf->type = type; + return 0; +} + +static int can_read(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener) +{ + int ret; + + spin_lock_bh(&dev->lock); + check_timers(dev); + ret = dev->write_position > opener->read_position || + dev->reread_count > opener->reread_count || dev->timeout_happened; + spin_unlock_bh(&dev->lock); + return ret; +} + +static int get_capture_buffer(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + int pos, timeout_happened; + u32 index; + + if ((file->f_flags & O_NONBLOCK) && + (dev->write_position <= opener->read_position && + dev->reread_count <= opener->reread_count && + !dev->timeout_happened)) + return -EAGAIN; + wait_event_interruptible(dev->read_event, can_read(dev, opener)); + + spin_lock_bh(&dev->lock); + if (dev->write_position == opener->read_position) { + if (dev->reread_count > opener->reread_count + 2) + opener->reread_count = dev->reread_count - 1; + ++opener->reread_count; + pos = v4l2l_mod64(opener->read_position + + dev->used_buffer_count - 1, + dev->used_buffer_count); + } else { + opener->reread_count = 0; + if (dev->write_position > + opener->read_position + dev->used_buffer_count) + opener->read_position = dev->write_position - 1; + pos = v4l2l_mod64(opener->read_position, + dev->used_buffer_count); + ++opener->read_position; + } + timeout_happened = dev->timeout_happened && (dev->timeout_jiffies > 0); + dev->timeout_happened = 0; + spin_unlock_bh(&dev->lock); + + index = dev->bufpos2index[pos]; + if (timeout_happened) { + if (index >= dev->used_buffer_count) { + dprintkrw("get_capture_buffer() read position is at " + "an unallocated buffer [index=%u]\n", + index); + return -EFAULT; + } + /* although allocated on-demand, timeout_image is freed only + * in free_buffers(), so we don't need to worry about it being + * deallocated suddenly */ + memcpy(dev->image + dev->buffers[index].buffer.m.offset, + dev->timeout_image, dev->buffer_size); + } + return (int)index; +} + +/* put buffer to dequeue + * called on VIDIOC_DQBUF + */ +static int vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 type = buf->type; + int index; + struct v4l2l_buffer *bufd; + + if (buf->memory != V4L2_MEMORY_MMAP) + return -EINVAL; + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + *buf = dev->timeout_buffer.buffer; + buf->type = type; + unset_flags(buf->flags); + return 0; + } + if ((opener->buffer_count == 0) || + !(opener->format_token & token_from_type(type))) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + index = get_capture_buffer(file); + if (index < 0) + return index; + *buf = dev->buffers[index].buffer; + unset_flags(buf->flags); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + spin_lock_bh(&dev->list_lock); + + bufd = list_first_entry_or_null(&dev->outbufs_list, + struct v4l2l_buffer, list_head); + if (bufd) + list_move_tail(&bufd->list_head, &dev->outbufs_list); + + spin_unlock_bh(&dev->list_lock); + if (!bufd) + return -EFAULT; + unset_flags(bufd->buffer.flags); + *buf = bufd->buffer; + break; + default: + return -EINVAL; + } + + buf->type = type; + dprintkrw("DQBUF(%s, index=%u) -> " BUFFER_DEBUG_FMT_STR, + V4L2_TYPE_IS_CAPTURE(type) ? "CAPTURE" : "OUTPUT", index, + BUFFER_DEBUG_FMT_ARGS(buf)); + return 0; +} + +/* ------------- STREAMING ------------------- */ + +/* start streaming + * called on VIDIOC_STREAMON + */ +static int vidioc_streamon(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = token_from_type(type); + + /* short-circuit when using timeout buffer set */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) + return 0; + /* opener must have claimed (same) buffer set via REQBUFS */ + if (!opener->buffer_count || !(opener->format_token & token)) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (has_output_token(dev->stream_tokens) && !dev->keep_format) + return -EIO; + if (dev->stream_tokens & token) { + acquire_token(dev, opener, stream, token); + client_usage_queue_event(dev->vdev); + } + return 0; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (dev->stream_tokens & token) + acquire_token(dev, opener, stream, token); + return 0; + default: + return -EINVAL; + } +} + +/* stop streaming + * called on VIDIOC_STREAMOFF + */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = token_from_type(type); + + /* short-circuit when using timeout buffer set */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) + return 0; + /* short-circuit when buffer set has no owner */ + if (dev->format_tokens & token) + return 0; + /* opener needs a claim to buffer set */ + if (!opener->format_token) + return -EBUSY; + if (opener->format_token & ~token) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (opener->stream_token & token) + release_token(dev, opener, stream); + /* reset output queue */ + if (dev->used_buffer_count > 0) + prepare_buffer_queue(dev, dev->used_buffer_count); + return 0; + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (opener->stream_token & token) { + release_token(dev, opener, stream); + client_usage_queue_event(dev->vdev); + } + return 0; + default: + return -EINVAL; + } +} + +#ifdef CONFIG_VIDEO_V4L1_COMPAT +static int vidiocgmbuf(struct file *file, void *fh, struct video_mbuf *p) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + p->frames = dev->buffer_count; + p->offsets[0] = 0; + p->offsets[1] = 0; + p->size = dev->buffer_size; + return 0; +} +#endif + +static void client_usage_queue_event(struct video_device *vdev) +{ + struct v4l2_event ev; + struct v4l2_loopback_device *dev; + + dev = container_of(vdev->v4l2_dev, struct v4l2_loopback_device, + v4l2_dev); + + memset(&ev, 0, sizeof(ev)); + ev.type = V4L2_EVENT_PRI_CLIENT_USAGE; + ((struct v4l2_event_client_usage *)&ev.u)->count = + !has_capture_token(dev->stream_tokens); + + v4l2_event_queue(vdev, &ev); +} + +static int client_usage_ops_add(struct v4l2_subscribed_event *sev, + unsigned elems) +{ + if (!(sev->flags & V4L2_EVENT_SUB_FL_SEND_INITIAL)) + return 0; + + client_usage_queue_event(sev->fh->vdev); + return 0; +} + +static void client_usage_ops_replace(struct v4l2_event *old, + const struct v4l2_event *new) +{ + *((struct v4l2_event_client_usage *)&old->u) = + *((struct v4l2_event_client_usage *)&new->u); +} + +static void client_usage_ops_merge(const struct v4l2_event *old, + struct v4l2_event *new) +{ + *((struct v4l2_event_client_usage *)&new->u) = + *((struct v4l2_event_client_usage *)&old->u); +} + +const struct v4l2_subscribed_event_ops client_usage_ops = { + .add = client_usage_ops_add, + .replace = client_usage_ops_replace, + .merge = client_usage_ops_merge, +}; + +static int vidioc_subscribe_event(struct v4l2_fh *fh, + const struct v4l2_event_subscription *sub) +{ + switch (sub->type) { + case V4L2_EVENT_CTRL: + return v4l2_ctrl_subscribe_event(fh, sub); + case V4L2_EVENT_PRI_CLIENT_USAGE: + return v4l2_event_subscribe(fh, sub, 0, &client_usage_ops); + } + + return -EINVAL; +} + +/* file operations */ +static void vm_open(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf; + MARK(); + + buf = vma->vm_private_data; + atomic_inc(&buf->use_count); + buf->buffer.flags |= V4L2_BUF_FLAG_MAPPED; +} + +static void vm_close(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf; + MARK(); + + buf = vma->vm_private_data; + if (atomic_dec_and_test(&buf->use_count)) + buf->buffer.flags &= ~V4L2_BUF_FLAG_MAPPED; +} + +static struct vm_operations_struct vm_ops = { + .open = vm_open, + .close = vm_close, +}; + +static int v4l2_loopback_mmap(struct file *file, struct vm_area_struct *vma) +{ + u8 *addr; + unsigned long start, size, offset; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + struct v4l2l_buffer *buffer = NULL; + int result = 0; + MARK(); + + offset = (unsigned long)vma->vm_pgoff << PAGE_SHIFT; + start = (unsigned long)vma->vm_start; + size = (unsigned long)(vma->vm_end - vma->vm_start); /* always != 0 */ + + /* ensure buffer size, count, and allocated image(s) are not altered by + * other file descriptors */ + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + + if (size > dev->buffer_size) { + dprintk("mmap() attempt to map %lubytes when %ubytes are " + "allocated to buffers\n", + size, dev->buffer_size); + result = -EINVAL; + goto exit_mmap_unlock; + } + if (offset % dev->buffer_size != 0) { + dprintk("mmap() offset does not match start of any buffer\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + switch (opener->format_token) { + case V4L2L_TOKEN_TIMEOUT: + if (offset != (unsigned long)dev->buffer_size * MAX_BUFFERS) { + dprintk("mmap() incorrect offset for timeout image\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + buffer = &dev->timeout_buffer; + addr = dev->timeout_image; + break; + default: + if (offset >= dev->image_size) { + dprintk("mmap() attempt to map beyond all buffers\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + u32 index = offset / dev->buffer_size; + buffer = &dev->buffers[index]; + addr = dev->image + offset; + break; + } + + while (size > 0) { + struct page *page = vmalloc_to_page(addr); + + result = vm_insert_page(vma, start, page); + if (result < 0) + goto exit_mmap_unlock; + + start += PAGE_SIZE; + addr += PAGE_SIZE; + size -= PAGE_SIZE; + } + + vma->vm_ops = &vm_ops; + vma->vm_private_data = buffer; + + vm_open(vma); +exit_mmap_unlock: + mutex_unlock(&dev->image_mutex); + return result; +} + +static unsigned int v4l2_loopback_poll(struct file *file, + struct poll_table_struct *pts) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + __poll_t req_events = poll_requested_events(pts); + int ret_mask = 0; + + /* call poll_wait in first call, regardless, to ensure that the + * wait-queue is not null */ + poll_wait(file, &dev->read_event, pts); + poll_wait(file, &opener->fh.wait, pts); + + if (req_events & POLLPRI) { + if (v4l2_event_pending(&opener->fh)) { + ret_mask |= POLLPRI; + if (!(req_events & DEFAULT_POLLMASK)) + return ret_mask; + } + } + + switch (opener->format_token) { + case V4L2L_TOKEN_OUTPUT: + if (opener->stream_token != 0 || + opener->io_method == V4L2L_IO_NONE) + ret_mask |= POLLOUT | POLLWRNORM; + break; + case V4L2L_TOKEN_CAPTURE: + if ((opener->io_method == V4L2L_IO_NONE || + opener->stream_token != 0) && + can_read(dev, opener)) + ret_mask |= POLLIN | POLLWRNORM; + break; + case V4L2L_TOKEN_TIMEOUT: + ret_mask |= POLLOUT | POLLWRNORM; + break; + default: + break; + } + + return ret_mask; +} + +/* do not want to limit device opens, it can be as many readers as user want, + * writers are limited by means of setting writer field */ +static int v4l2_loopback_open(struct file *file) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + + dev = v4l2loopback_getdevice(file); + if (dev->open_count.counter >= dev->max_openers) + return -EBUSY; + /* kfree on close */ + opener = kzalloc(sizeof(*opener), GFP_KERNEL); + if (opener == NULL) + return -ENOMEM; + + atomic_inc(&dev->open_count); + if (dev->timeout_image_io && dev->format_tokens & V4L2L_TOKEN_TIMEOUT) + /* will clear timeout_image_io once buffer set acquired */ + opener->io_method = V4L2L_IO_TIMEOUT; + + v4l2_fh_init(&opener->fh, video_devdata(file)); + file->private_data = &opener->fh; + + v4l2_fh_add(&opener->fh); + dprintk("open() -> dev@%p with image@%p\n", dev, + dev ? dev->image : NULL); + return 0; +} + +static int v4l2_loopback_close(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + int result = 0; + dprintk("close() -> dev@%p with image@%p\n", dev, + dev ? dev->image : NULL); + + if (opener->format_token) { + struct v4l2_requestbuffers reqbuf = { + .count = 0, .memory = V4L2_MEMORY_MMAP, .type = 0 + }; + switch (opener->format_token) { + case V4L2L_TOKEN_CAPTURE: + reqbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + break; + case V4L2L_TOKEN_OUTPUT: + case V4L2L_TOKEN_TIMEOUT: + reqbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + break; + } + if (reqbuf.type) + result = vidioc_reqbufs(file, file->private_data, + &reqbuf); + if (result < 0) + dprintk("failed to free buffers REQBUFS(count=0) " + " returned %d\n", + result); + mutex_lock(&dev->image_mutex); + release_token(dev, opener, format); + mutex_unlock(&dev->image_mutex); + } + + if (atomic_dec_and_test(&dev->open_count)) { + timer_delete_sync(&dev->sustain_timer); + timer_delete_sync(&dev->timeout_timer); + if (!dev->keep_format) { + mutex_lock(&dev->image_mutex); + free_buffers(dev); + mutex_unlock(&dev->image_mutex); + } + } + + v4l2_fh_del(&opener->fh); + v4l2_fh_exit(&opener->fh); + + kfree(opener); + return 0; +} + +static int start_fileio(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + struct v4l2_requestbuffers reqbuf = { .count = dev->buffer_count, + .memory = V4L2_MEMORY_MMAP, + .type = type }; + int token = token_from_type(type); + int result; + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT || + opener->format_token & ~token) + return -EBUSY; /* NOTE: -EBADF might be more informative */ + + /* short-circuit if already have stream token */ + if (opener->stream_token && opener->io_method == V4L2L_IO_FILE) + return 0; + + /* otherwise attempt to acquire stream token and assign IO method */ + if (!(dev->stream_tokens & token) || opener->io_method != V4L2L_IO_NONE) + return -EBUSY; + + result = vidioc_reqbufs(file, fh, &reqbuf); + if (result < 0) + return result; + result = vidioc_streamon(file, fh, type); + if (result < 0) + return result; + + opener->io_method = V4L2L_IO_FILE; + return 0; +} + +static ssize_t v4l2_loopback_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_buffer *b; + int index, result; + + dprintkrw("read() %zu bytes\n", count); + result = start_fileio(file, file->private_data, + V4L2_BUF_TYPE_VIDEO_CAPTURE); + if (result < 0) + return result; + + index = get_capture_buffer(file); + if (index < 0) + return index; + b = &dev->buffers[index].buffer; + if (count > b->bytesused) + count = b->bytesused; + if (copy_to_user((void *)buf, (void *)(dev->image + b->m.offset), + count)) { + printk(KERN_ERR "v4l2-loopback read() failed copy_to_user()\n"); + return -EFAULT; + } + return count; +} + +static ssize_t v4l2_loopback_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_buffer *b; + int index, result; + + dprintkrw("write() %zu bytes\n", count); + result = start_fileio(file, file->private_data, + V4L2_BUF_TYPE_VIDEO_OUTPUT); + if (result < 0) + return result; + + if (count > dev->buffer_size) + count = dev->buffer_size; + index = v4l2l_mod64(dev->write_position, dev->used_buffer_count); + b = &dev->buffers[index].buffer; + + if (copy_from_user((void *)(dev->image + b->m.offset), (void *)buf, + count)) { + printk(KERN_ERR + "v4l2-loopback write() failed copy_from_user()\n"); + return -EFAULT; + } + b->bytesused = count; + + v4l2l_get_timestamp(b); + b->sequence = dev->write_position; + set_queued(b->flags); + buffer_written(dev, &dev->buffers[index]); + set_done(b->flags); + wake_up_all(&dev->read_event); + + return count; +} + +/* init functions */ +/* frees buffers, if allocated */ +static void free_buffers(struct v4l2_loopback_device *dev) +{ + dprintk("free_buffers() with image@%p\n", dev->image); + if (!dev->image) + return; + if (!has_no_owners(dev) || any_buffers_mapped(dev)) + /* maybe an opener snuck in before image_mutex was acquired */ + printk(KERN_WARNING + "v4l2-loopback free_buffers() buffers of video device " + "#%u freed while still mapped to userspace\n", + dev->vdev->num); + vfree(dev->image); + dev->image = NULL; + dev->image_size = 0; + dev->buffer_size = 0; +} + +static void free_timeout_buffer(struct v4l2_loopback_device *dev) +{ + dprintk("free_timeout_buffer() with timeout_image@%p\n", + dev->timeout_image); + if (!dev->timeout_image) + return; + + if ((dev->timeout_jiffies > 0 && !has_no_owners(dev)) || + dev->timeout_buffer.buffer.flags & V4L2_BUF_FLAG_MAPPED) + printk(KERN_WARNING + "v4l2-loopback free_timeout_buffer() timeout image " + "of device #%u freed while still mapped to userspace\n", + dev->vdev->num); + + vfree(dev->timeout_image); + dev->timeout_image = NULL; + dev->timeout_buffer_size = 0; +} +/* allocates buffers if no (other) openers are already using them */ +static int allocate_buffers(struct v4l2_loopback_device *dev, + struct v4l2_pix_format *pix_format) +{ + u32 buffer_size = PAGE_ALIGN(pix_format->sizeimage); + unsigned long image_size = + (unsigned long)buffer_size * (unsigned long)dev->buffer_count; + /* vfree on close file operation in case no open handles left */ + + if (buffer_size == 0 || dev->buffer_count == 0 || + buffer_size < pix_format->sizeimage) + return -EINVAL; + + if ((__LONG_MAX__ / buffer_size) < dev->buffer_count) + return -ENOSPC; + + dprintk("allocate_buffers() size %lubytes = %ubytes x %ubuffers\n", + image_size, buffer_size, dev->buffer_count); + if (dev->image) { + /* check that no buffers are expected in user-space */ + if (!has_no_owners(dev) || any_buffers_mapped(dev)) + return -EBUSY; + dprintk("allocate_buffers() existing size=%lubytes\n", + dev->image_size); + /* FIXME: prevent double allocation more intelligently! */ + if (image_size == dev->image_size) { + dprintk("allocate_buffers() keep existing\n"); + return 0; + } + free_buffers(dev); + } + + /* FIXME: set buffers to 0 */ + dev->image = vmalloc(image_size); + if (dev->image == NULL) { + dev->buffer_size = dev->image_size = 0; + return -ENOMEM; + } + init_buffers(dev, pix_format->sizeimage, buffer_size); + dev->buffer_size = buffer_size; + dev->image_size = image_size; + dprintk("allocate_buffers() -> vmalloc'd %lubytes\n", dev->image_size); + return 0; +} +static int allocate_timeout_buffer(struct v4l2_loopback_device *dev) +{ + /* device's `buffer_size` and `buffers` must be initialised in + * allocate_buffers() */ + + dprintk("allocate_timeout_buffer() size %ubytes\n", dev->buffer_size); + if (dev->buffer_size == 0) + return -EINVAL; + + if (dev->timeout_image) { + if (dev->timeout_buffer.buffer.flags & V4L2_BUF_FLAG_MAPPED) + return -EBUSY; + if (dev->buffer_size == dev->timeout_buffer_size) + return 0; + free_timeout_buffer(dev); + } + + dev->timeout_image = vzalloc(dev->buffer_size); + if (!dev->timeout_image) { + dev->timeout_buffer_size = 0; + return -ENOMEM; + } + dev->timeout_buffer_size = dev->buffer_size; + return 0; +} +/* init inner buffers, they are capture mode and flags are set as for capture + * mode buffers */ +static void init_buffers(struct v4l2_loopback_device *dev, u32 bytes_used, + u32 buffer_size) +{ + u32 i; + + for (i = 0; i < dev->buffer_count; ++i) { + struct v4l2_buffer *b = &dev->buffers[i].buffer; + b->index = i; + b->bytesused = bytes_used; + b->length = buffer_size; + b->field = V4L2_FIELD_NONE; + b->flags = 0; + b->m.offset = i * buffer_size; + b->memory = V4L2_MEMORY_MMAP; + b->sequence = 0; + b->timestamp.tv_sec = 0; + b->timestamp.tv_usec = 0; + b->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + v4l2l_get_timestamp(b); + } + dev->timeout_buffer = dev->buffers[0]; + dev->timeout_buffer.buffer.m.offset = MAX_BUFFERS * buffer_size; +} + +/* fills and register video device */ +static void init_vdev(struct video_device *vdev, int nr) +{ +#ifdef V4L2LOOPBACK_WITH_STD + vdev->tvnorms = V4L2_STD_ALL; +#endif /* V4L2LOOPBACK_WITH_STD */ + + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->fops = &v4l2_loopback_fops; + vdev->ioctl_ops = &v4l2_loopback_ioctl_ops; + vdev->release = &video_device_release; + vdev->minor = -1; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + vdev->device_caps = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_VIDEO_CAPTURE | + V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_READWRITE | + V4L2_CAP_STREAMING; +#endif + + if (debug > 1) + vdev->dev_debug = V4L2_DEV_DEBUG_IOCTL | + V4L2_DEV_DEBUG_IOCTL_ARG; + + vdev->vfl_dir = VFL_DIR_M2M; +} + +/* init default capture parameters, only fps may be changed in future */ +static void init_capture_param(struct v4l2_captureparm *capture_param) +{ + capture_param->capability = V4L2_CAP_TIMEPERFRAME; /* since 2.16 */ + capture_param->capturemode = 0; + capture_param->extendedmode = 0; + capture_param->readbuffers = max_buffers; + capture_param->timeperframe.numerator = 1; + capture_param->timeperframe.denominator = V4L2LOOPBACK_FPS_DEFAULT; +} + +static void check_timers(struct v4l2_loopback_device *dev) +{ + if (has_output_token(dev->stream_tokens)) + return; + + if (dev->timeout_jiffies > 0 && !timer_pending(&dev->timeout_timer)) + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + if (dev->sustain_framerate && !timer_pending(&dev->sustain_timer)) + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies * 3 / 2); +} +#ifdef HAVE_TIMER_SETUP +static void sustain_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = timer_container_of(dev, t, sustain_timer); +#else +static void sustain_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->sustain_framerate) { + dev->reread_count++; + dprintkrw("sustain_timer_clb() write_pos=%lld reread=%u\n", + (long long)dev->write_position, dev->reread_count); + if (dev->reread_count == 1) + mod_timer(&dev->sustain_timer, + jiffies + max(1UL, dev->frame_jiffies / 2)); + else + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} +#ifdef HAVE_TIMER_SETUP +static void timeout_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = timer_container_of(dev, t, timeout_timer); +#else +static void timeout_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->timeout_jiffies > 0) { + dev->timeout_happened = 1; + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} + +/* init loopback main structure */ +#define DEFAULT_FROM_CONF(confmember, default_condition, default_value) \ + ((conf) ? \ + ((conf->confmember default_condition) ? (default_value) : \ + (conf->confmember)) : \ + default_value) + +static int v4l2_loopback_add(struct v4l2_loopback_config *conf, int *ret_nr) +{ + struct v4l2_loopback_device *dev; + struct v4l2_ctrl_handler *hdl; + struct v4l2loopback_private *vdev_priv = NULL; + int err; + + u32 _width = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + u32 _height = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + + u32 _min_width = DEFAULT_FROM_CONF(min_width, + < V4L2LOOPBACK_SIZE_MIN_WIDTH, + V4L2LOOPBACK_SIZE_MIN_WIDTH); + u32 _min_height = DEFAULT_FROM_CONF(min_height, + < V4L2LOOPBACK_SIZE_MIN_HEIGHT, + V4L2LOOPBACK_SIZE_MIN_HEIGHT); + u32 _max_width = DEFAULT_FROM_CONF(max_width, < _min_width, max_width); + u32 _max_height = + DEFAULT_FROM_CONF(max_height, < _min_height, max_height); + bool _announce_all_caps = (conf && conf->announce_all_caps >= 0) ? + (bool)(conf->announce_all_caps) : + !(V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS); + int _max_buffers = DEFAULT_FROM_CONF(max_buffers, <= 0, max_buffers); + int _max_openers = DEFAULT_FROM_CONF(max_openers, <= 0, max_openers); + struct v4l2_format _fmt; + + int nr = -1; + + if (conf) { + const int output_nr = conf->output_nr; +#ifdef SPLIT_DEVICES + const int capture_nr = conf->capture_nr; +#else + const int capture_nr = output_nr; +#endif + if (capture_nr >= 0 && output_nr == capture_nr) { + nr = output_nr; + } else if (capture_nr < 0 && output_nr < 0) { + nr = -1; + } else if (capture_nr < 0) { + nr = output_nr; + } else if (output_nr < 0) { + nr = capture_nr; + } else { + printk(KERN_ERR + "v4l2-loopback add() split OUTPUT and CAPTURE " + "devices not yet supported.\n"); + printk(KERN_INFO + "v4l2-loopback add() both devices must have the " + "same number (%d != %d).\n", + output_nr, capture_nr); + return -EINVAL; + } + } + + if (idr_find(&v4l2loopback_index_idr, nr)) + return -EEXIST; + + /* initialisation of a new device */ + dprintk("add() creating device #%d\n", nr); + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + /* allocate id, if @id >= 0, we're requesting that specific id */ + if (nr >= 0) { + err = idr_alloc(&v4l2loopback_index_idr, dev, nr, nr + 1, + GFP_KERNEL); + if (err == -ENOSPC) + err = -EEXIST; + } else { + err = idr_alloc(&v4l2loopback_index_idr, dev, 0, 0, GFP_KERNEL); + } + if (err < 0) + goto out_free_dev; + + /* register new device */ + MARK(); + nr = err; + + if (conf && conf->card_label[0]) { + snprintf(dev->card_label, sizeof(dev->card_label), "%s", + conf->card_label); + } else { + snprintf(dev->card_label, sizeof(dev->card_label), + "Dummy video device (0x%04X)", nr); + } + snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), + "v4l2loopback-%03d", nr); + + err = v4l2_device_register(NULL, &dev->v4l2_dev); + if (err) + goto out_free_idr; + + /* initialise the _video_ device */ + MARK(); + err = -ENOMEM; + dev->vdev = video_device_alloc(); + if (dev->vdev == NULL) + goto out_unregister; + + vdev_priv = kzalloc(sizeof(struct v4l2loopback_private), GFP_KERNEL); + if (vdev_priv == NULL) + goto out_unregister; + + video_set_drvdata(dev->vdev, vdev_priv); + if (video_get_drvdata(dev->vdev) == NULL) + goto out_unregister; + + snprintf(dev->vdev->name, sizeof(dev->vdev->name), "%s", + dev->card_label); + vdev_priv->device_nr = nr; + init_vdev(dev->vdev, nr); + dev->vdev->v4l2_dev = &dev->v4l2_dev; + + /* initialise v4l2-loopback specific parameters */ + MARK(); + dev->announce_all_caps = _announce_all_caps; + dev->min_width = _min_width; + dev->min_height = _min_height; + dev->max_width = _max_width; + dev->max_height = _max_height; + dev->max_openers = _max_openers; + + /* set (initial) pixel and stream format */ + _width = clamp_val(_width, _min_width, _max_width); + _height = clamp_val(_height, _min_height, _max_height); + _fmt = (struct v4l2_format){ + .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, + .fmt.pix = { .width = _width, + .height = _height, + .pixelformat = formats[0].fourcc, + .colorspace = V4L2_COLORSPACE_DEFAULT, + .field = V4L2_FIELD_NONE } + }; + + err = v4l2l_fill_format(&_fmt, _min_width, _max_width, _min_height, + _max_height); + if (err) + /* highly unexpected failure to assign default format */ + goto out_unregister; + dev->pix_format = _fmt.fmt.pix; + init_capture_param(&dev->capture_param); + set_timeperframe(dev, &dev->capture_param.timeperframe); + + /* ctrls parameters */ + dev->keep_format = 0; + dev->sustain_framerate = 0; + dev->timeout_jiffies = 0; + dev->timeout_image_io = 0; + + /* initialise OUTPUT and CAPTURE buffer values */ + dev->image = NULL; + dev->image_size = 0; + dev->buffer_count = _max_buffers; + dev->buffer_size = 0; + dev->used_buffer_count = 0; + INIT_LIST_HEAD(&dev->outbufs_list); + do { + u32 index; + for (index = 0; index < dev->buffer_count; ++index) + INIT_LIST_HEAD(&dev->buffers[index].list_head); + + } while (0); + memset(dev->bufpos2index, 0, sizeof(dev->bufpos2index)); + dev->write_position = 0; + + /* initialise synchronisation data */ + atomic_set(&dev->open_count, 0); + mutex_init(&dev->image_mutex); + spin_lock_init(&dev->lock); + spin_lock_init(&dev->list_lock); + init_waitqueue_head(&dev->read_event); + dev->format_tokens = V4L2L_TOKEN_MASK; + dev->stream_tokens = V4L2L_TOKEN_MASK; + + /* initialise sustain frame rate and timeout parameters, and timers */ + dev->reread_count = 0; + dev->timeout_image = NULL; + dev->timeout_happened = 0; +#ifdef HAVE_TIMER_SETUP + timer_setup(&dev->sustain_timer, sustain_timer_clb, 0); + timer_setup(&dev->timeout_timer, timeout_timer_clb, 0); +#else + setup_timer(&dev->sustain_timer, sustain_timer_clb, nr); + setup_timer(&dev->timeout_timer, timeout_timer_clb, nr); +#endif + + /* initialise the control handler and add controls */ + MARK(); + hdl = &dev->ctrl_handler; + err = v4l2_ctrl_handler_init(hdl, 4); + if (err) + goto out_unregister; + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_keepformat, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_sustainframerate, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeout, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeoutimageio, NULL); + if (hdl->error) { + err = hdl->error; + goto out_free_handler; + } + dev->v4l2_dev.ctrl_handler = hdl; + + err = v4l2_ctrl_handler_setup(hdl); + if (err) + goto out_free_handler; + + /* register the device (creates /dev/video*) */ + MARK(); + if (video_register_device(dev->vdev, VFL_TYPE_VIDEO, nr) < 0) { + printk(KERN_ERR + "v4l2-loopback add() failed video_register_device()\n"); + err = -EFAULT; + goto out_free_device; + } + v4l2loopback_create_sysfs(dev->vdev); + /* NOTE: ambivalent if sysfs entries fail */ + + if (ret_nr) + *ret_nr = dev->vdev->num; + return 0; + +out_free_device: + video_device_release(dev->vdev); +out_free_handler: + v4l2_ctrl_handler_free(&dev->ctrl_handler); +out_unregister: + video_set_drvdata(dev->vdev, NULL); + if (vdev_priv != NULL) + kfree(vdev_priv); + v4l2_device_unregister(&dev->v4l2_dev); +out_free_idr: + idr_remove(&v4l2loopback_index_idr, nr); +out_free_dev: + kfree(dev); + return err; +} + +static void v4l2_loopback_remove(struct v4l2_loopback_device *dev) +{ + int device_nr = v4l2loopback_get_vdev_nr(dev->vdev); + mutex_lock(&dev->image_mutex); + free_buffers(dev); + free_timeout_buffer(dev); + mutex_unlock(&dev->image_mutex); + v4l2loopback_remove_sysfs(dev->vdev); + v4l2_ctrl_handler_free(&dev->ctrl_handler); + kfree(video_get_drvdata(dev->vdev)); + video_unregister_device(dev->vdev); + v4l2_device_unregister(&dev->v4l2_dev); + idr_remove(&v4l2loopback_index_idr, device_nr); + kfree(dev); +} + +static long v4l2loopback_control_ioctl(struct file *file, unsigned int cmd, + unsigned long parm) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_config conf; + struct v4l2_loopback_config *confptr = &conf; + int device_nr, capture_nr, output_nr; + int ret; + const __u32 version = V4L2LOOPBACK_VERSION_CODE; + + ret = mutex_lock_killable(&v4l2loopback_ctl_mutex); + if (ret) + return ret; + + ret = -EINVAL; + switch (cmd) { + default: + ret = -ENOSYS; + break; + /* add a v4l2loopback device (pair), based on the user-provided specs */ + case V4L2LOOPBACK_CTL_ADD: + case V4L2LOOPBACK_CTL_ADD_legacy: + if (parm) { + if ((ret = copy_from_user(&conf, (void *)parm, + sizeof(conf))) < 0) + break; + } else + confptr = NULL; + ret = v4l2_loopback_add(confptr, &device_nr); + if (ret >= 0) + ret = device_nr; + break; + /* remove a v4l2loopback device (both capture and output) */ + case V4L2LOOPBACK_CTL_REMOVE: + case V4L2LOOPBACK_CTL_REMOVE_legacy: + ret = v4l2loopback_lookup((__u32)parm, &dev); + if (ret >= 0 && dev) { + ret = -EBUSY; + if (dev->open_count.counter > 0) + break; + v4l2_loopback_remove(dev); + ret = 0; + }; + break; + /* get information for a loopback device. + * this is mostly about limits (which cannot be queried directly with VIDIOC_G_FMT and friends + */ + case V4L2LOOPBACK_CTL_QUERY: + case V4L2LOOPBACK_CTL_QUERY_legacy: + if (!parm) + break; + if ((ret = copy_from_user(&conf, (void *)parm, sizeof(conf))) < + 0) + break; + capture_nr = output_nr = conf.output_nr; +#ifdef SPLIT_DEVICES + capture_nr = conf.capture_nr; +#endif + device_nr = (output_nr < 0) ? capture_nr : output_nr; + MARK(); + /* get the device from either capture_nr or output_nr (whatever is valid) */ + if ((ret = v4l2loopback_lookup(device_nr, &dev)) < 0) + break; + MARK(); + /* if we got the device from output_nr and there is a valid capture_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != capture_nr) && (capture_nr >= 0) && + ((ret = v4l2loopback_lookup(capture_nr, 0)) < 0)) + break; + MARK(); + /* if otoh, we got the device from capture_nr and there is a valid output_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != output_nr) && (output_nr >= 0) && + ((ret = v4l2loopback_lookup(output_nr, 0)) < 0)) + break; + + /* v4l2_loopback_config identified a single device, so fetch the data */ + snprintf(conf.card_label, sizeof(conf.card_label), "%s", + dev->card_label); + + conf.output_nr = dev->vdev->num; +#ifdef SPLIT_DEVICES + conf.capture_nr = dev->vdev->num; +#endif + conf.min_width = dev->min_width; + conf.min_height = dev->min_height; + conf.max_width = dev->max_width; + conf.max_height = dev->max_height; + conf.announce_all_caps = dev->announce_all_caps; + conf.max_buffers = dev->buffer_count; + conf.max_openers = dev->max_openers; + conf.debug = debug; + MARK(); + if (copy_to_user((void *)parm, &conf, sizeof(conf))) { + ret = -EFAULT; + break; + } + ret = 0; + break; + case V4L2LOOPBACK_CTL_VERSION: + if (!parm) + break; + if (copy_to_user((void *)parm, &version, sizeof(version))) { + ret = -EFAULT; + break; + } + ret = 0; + break; + } + + mutex_unlock(&v4l2loopback_ctl_mutex); + MARK(); + return ret; +} + +/* LINUX KERNEL */ + +static const struct file_operations v4l2loopback_ctl_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = nonseekable_open, + .unlocked_ioctl = v4l2loopback_control_ioctl, + .compat_ioctl = v4l2loopback_control_ioctl, + .llseek = noop_llseek, + // clang-format on +}; + +static struct miscdevice v4l2loopback_misc = { + // clang-format off + .minor = MISC_DYNAMIC_MINOR, + .name = "v4l2loopback", + .fops = &v4l2loopback_ctl_fops, + // clang-format on +}; + +static const struct v4l2_file_operations v4l2_loopback_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = v4l2_loopback_open, + .release = v4l2_loopback_close, + .read = v4l2_loopback_read, + .write = v4l2_loopback_write, + .poll = v4l2_loopback_poll, + .mmap = v4l2_loopback_mmap, + .unlocked_ioctl = video_ioctl2, + // clang-format on +}; + +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops = { + // clang-format off + .vidioc_querycap = &vidioc_querycap, + .vidioc_enum_framesizes = &vidioc_enum_framesizes, + .vidioc_enum_frameintervals = &vidioc_enum_frameintervals, + + .vidioc_enum_output = &vidioc_enum_output, + .vidioc_g_output = &vidioc_g_output, + .vidioc_s_output = &vidioc_s_output, + + .vidioc_enum_input = &vidioc_enum_input, + .vidioc_g_input = &vidioc_g_input, + .vidioc_s_input = &vidioc_s_input, + + .vidioc_enum_fmt_vid_cap = &vidioc_enum_fmt_cap, + .vidioc_g_fmt_vid_cap = &vidioc_g_fmt_cap, + .vidioc_s_fmt_vid_cap = &vidioc_s_fmt_cap, + .vidioc_try_fmt_vid_cap = &vidioc_try_fmt_cap, + + .vidioc_enum_fmt_vid_out = &vidioc_enum_fmt_out, + .vidioc_s_fmt_vid_out = &vidioc_s_fmt_out, + .vidioc_g_fmt_vid_out = &vidioc_g_fmt_out, + .vidioc_try_fmt_vid_out = &vidioc_try_fmt_out, + +#ifdef V4L2L_OVERLAY + .vidioc_s_fmt_vid_overlay = &vidioc_s_fmt_overlay, + .vidioc_g_fmt_vid_overlay = &vidioc_g_fmt_overlay, +#endif + +#ifdef V4L2LOOPBACK_WITH_STD + .vidioc_s_std = &vidioc_s_std, + .vidioc_g_std = &vidioc_g_std, + .vidioc_querystd = &vidioc_querystd, +#endif /* V4L2LOOPBACK_WITH_STD */ + + .vidioc_g_parm = &vidioc_g_parm, + .vidioc_s_parm = &vidioc_s_parm, + + .vidioc_reqbufs = &vidioc_reqbufs, + .vidioc_querybuf = &vidioc_querybuf, + .vidioc_qbuf = &vidioc_qbuf, + .vidioc_dqbuf = &vidioc_dqbuf, + + .vidioc_streamon = &vidioc_streamon, + .vidioc_streamoff = &vidioc_streamoff, + +#ifdef CONFIG_VIDEO_V4L1_COMPAT + .vidiocgmbuf = &vidiocgmbuf, +#endif + + .vidioc_subscribe_event = &vidioc_subscribe_event, + .vidioc_unsubscribe_event = &v4l2_event_unsubscribe, + // clang-format on +}; + +static int free_device_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *dev = ptr; + v4l2_loopback_remove(dev); + return 0; +} +static void free_devices(void) +{ + idr_for_each(&v4l2loopback_index_idr, &free_device_cb, NULL); + idr_destroy(&v4l2loopback_index_idr); +} + +static int __init v4l2loopback_init_module(void) +{ + const u32 min_width = V4L2LOOPBACK_SIZE_MIN_WIDTH; + const u32 min_height = V4L2LOOPBACK_SIZE_MIN_HEIGHT; + int err; + int i; + MARK(); + + err = misc_register(&v4l2loopback_misc); + if (err < 0) + return err; + + if (devices < 0) { + devices = 1; + + /* try guessing the devices from the "video_nr" parameter */ + for (i = MAX_DEVICES - 1; i >= 0; i--) { + if (video_nr[i] >= 0) { + devices = i + 1; + break; + } + } + } + + if (devices > MAX_DEVICES) { + devices = MAX_DEVICES; + printk(KERN_INFO + "v4l2-loopback init() number of initial devices is " + "limited to: %d\n", + MAX_DEVICES); + } + + if (max_buffers > MAX_BUFFERS) { + max_buffers = MAX_BUFFERS; + printk(KERN_INFO + "v4l2-loopback init() number of buffers is limited " + "to: %d\n", + MAX_BUFFERS); + } + + if (max_openers < 0) { + printk(KERN_INFO + "v4l2-loopback init() allowing %d openers rather " + "than %d\n", + 2, max_openers); + max_openers = 2; + } + + if (max_width < min_width) { + max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; + printk(KERN_INFO "v4l2-loopback init() using max_width %d\n", + max_width); + } + if (max_height < min_height) { + max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; + printk(KERN_INFO "v4l2-loopback init() using max_height %d\n", + max_height); + } + + for (i = 0; i < devices; i++) { + struct v4l2_loopback_config cfg = { + // clang-format off + .output_nr = video_nr[i], +#ifdef SPLIT_DEVICES + .capture_nr = video_nr[i], +#endif + .min_width = min_width, + .min_height = min_height, + .max_width = max_width, + .max_height = max_height, + .announce_all_caps = (!exclusive_caps[i]), + .max_buffers = max_buffers, + .max_openers = max_openers, + .debug = debug, + // clang-format on + }; + cfg.card_label[0] = 0; + if (card_label[i]) + snprintf(cfg.card_label, sizeof(cfg.card_label), "%s", + card_label[i]); + err = v4l2_loopback_add(&cfg, 0); + if (err) { + free_devices(); + goto error; + } + } + + dprintk("module installed\n"); + + printk(KERN_INFO "v4l2-loopback driver version %d.%d.%d%s loaded\n", + // clang-format off + (V4L2LOOPBACK_VERSION_CODE >> 16) & 0xff, + (V4L2LOOPBACK_VERSION_CODE >> 8) & 0xff, + (V4L2LOOPBACK_VERSION_CODE ) & 0xff, +#ifdef SNAPSHOT_VERSION + " (" __stringify(SNAPSHOT_VERSION) ")" +#else + "" +#endif + ); + // clang-format on + + return 0; +error: + misc_deregister(&v4l2loopback_misc); + return err; +} + +static void v4l2loopback_cleanup_module(void) +{ + MARK(); + /* unregister the device -> it deletes /dev/video* */ + free_devices(); + /* and get rid of /dev/v4l2loopback */ + misc_deregister(&v4l2loopback_misc); + dprintk("module removed\n"); +} + +MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR); + +module_init(v4l2loopback_init_module); +module_exit(v4l2loopback_cleanup_module); diff --git a/drivers/media/v4l2-core/v4l2loopback.h b/drivers/media/v4l2-core/v4l2loopback.h new file mode 100644 index 000000000000..e48e0ce5949d --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * v4l2loopback.h + * + * Written by IOhannes m zmölnig, 7/1/20. + * + * Copyright 2020 by IOhannes m zmölnig. Redistribution of this file is + * permitted under the GNU General Public License. + */ +#ifndef _V4L2LOOPBACK_H +#define _V4L2LOOPBACK_H + +#define V4L2LOOPBACK_VERSION_MAJOR 0 +#define V4L2LOOPBACK_VERSION_MINOR 15 +#define V4L2LOOPBACK_VERSION_BUGFIX 0 + +/* /dev/v4l2loopback interface */ + +struct v4l2_loopback_config { + /** + * the device-number (/dev/video) + * V4L2LOOPBACK_CTL_ADD: + * setting this to a value<0, will allocate an available one + * if nr>=0 and the device already exists, the ioctl will EEXIST + * if output_nr and capture_nr are the same, only a single device will be created + * NOTE: currently split-devices (where output_nr and capture_nr differ) + * are not implemented yet. + * until then, requesting different device-IDs will result in EINVAL. + * + * V4L2LOOPBACK_CTL_QUERY: + * either both output_nr and capture_nr must refer to the same loopback, + * or one (and only one) of them must be -1 + * + */ + __s32 output_nr; + __s32 unused; /*capture_nr;*/ + + /** + * a nice name for your device + * if (*card_label)==0, an automatic name is assigned + */ + char card_label[32]; + + /** + * allowed frame size + * if too low, default values are used + */ + __u32 min_width; + __u32 max_width; + __u32 min_height; + __u32 max_height; + + /** + * number of buffers to allocate for the queue + * if set to <=0, default values are used + */ + __s32 max_buffers; + + /** + * how many consumers are allowed to open this device concurrently + * if set to <=0, default values are used + */ + __s32 max_openers; + + /** + * set the debugging level for this device + */ + __s32 debug; + + /** + * whether to announce OUTPUT/CAPTURE capabilities exclusively + * for this device or not + * (!exclusive_caps) + * NOTE: this is going to be removed once separate output/capture + * devices are implemented + */ + __s32 announce_all_caps; +}; + +#define V4L2LOOPBACK_CTL_IOCTLMAGIC '~' + +/* a pointer to an (unsigned int) that - on success - will hold + * the version code of the v4l2loopback module + * as returned by KERNEL_VERSION(MAJOR, MINOR, BUGFIX) + */ +#define V4L2LOOPBACK_CTL_VERSION _IOR(V4L2LOOPBACK_CTL_IOCTLMAGIC, 0, __u32) + +/* a pointer to a (struct v4l2_loopback_config) that has all values you wish to impose on the + * to-be-created device set. + * if the ptr is NULL, a new device is created with default values at the driver's discretion. + * + * returns the device_nr of the OUTPUT device (which can be used with V4L2LOOPBACK_CTL_QUERY, + * to get more information on the device) + */ +#define V4L2LOOPBACK_CTL_ADD \ + _IOW(V4L2LOOPBACK_CTL_IOCTLMAGIC, 1, struct v4l2_loopback_config) + +/* the device-number (either CAPTURE or OUTPUT) associated with the loopback-device */ +#define V4L2LOOPBACK_CTL_REMOVE _IOW(V4L2LOOPBACK_CTL_IOCTLMAGIC, 2, __u32) + +/* a pointer to a (struct v4l2_loopback_config) that has output_nr and/or capture_nr set + * (the two values must either refer to video-devices associated with the same loopback device + * or exactly one of them must be <0 + */ +#define V4L2LOOPBACK_CTL_QUERY \ + _IOWR(V4L2LOOPBACK_CTL_IOCTLMAGIC, 3, struct v4l2_loopback_config) + +#endif /* _V4L2LOOPBACK_H */ diff --git a/drivers/media/v4l2-core/v4l2loopback_formats.h b/drivers/media/v4l2-core/v4l2loopback_formats.h new file mode 100644 index 000000000000..d855a3796554 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback_formats.h @@ -0,0 +1,445 @@ +static const struct v4l2l_format formats[] = { +#ifndef V4L2_PIX_FMT_VP9 +#define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0') +#endif +#ifndef V4L2_PIX_FMT_HEVC +#define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C') +#endif + + /* here come the packed formats */ + { + .name = "32 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR32, + .depth = 32, + .flags = 0, + }, + { + .name = "32 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB32, + .depth = 32, + .flags = 0, + }, + { + .name = "24 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR24, + .depth = 24, + .flags = 0, + }, + { + .name = "24 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB24, + .depth = 24, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_ABGR32 + { + .name = "32 bpp RGBA, le", + .fourcc = V4L2_PIX_FMT_ABGR32, + .depth = 32, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_RGBA32 + { + .name = "32 bpp RGBA", + .fourcc = V4L2_PIX_FMT_RGBA32, + .depth = 32, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_RGB332 + { + .name = "8 bpp RGB-3-3-2", + .fourcc = V4L2_PIX_FMT_RGB332, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB332 */ +#ifdef V4L2_PIX_FMT_RGB444 + { + .name = "16 bpp RGB (xxxxrrrr ggggbbbb)", + .fourcc = V4L2_PIX_FMT_RGB444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB444 */ +#ifdef V4L2_PIX_FMT_RGB555 + { + .name = "16 bpp RGB-5-5-5", + .fourcc = V4L2_PIX_FMT_RGB555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555 */ +#ifdef V4L2_PIX_FMT_RGB565 + { + .name = "16 bpp RGB-5-6-5", + .fourcc = V4L2_PIX_FMT_RGB565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565 */ +#ifdef V4L2_PIX_FMT_RGB555X + { + .name = "16 bpp RGB-5-5-5 BE", + .fourcc = V4L2_PIX_FMT_RGB555X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555X */ +#ifdef V4L2_PIX_FMT_RGB565X + { + .name = "16 bpp RGB-5-6-5 BE", + .fourcc = V4L2_PIX_FMT_RGB565X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565X */ +#ifdef V4L2_PIX_FMT_BGR666 + { + .name = "18 bpp BGR-6-6-6", + .fourcc = V4L2_PIX_FMT_BGR666, + .depth = 18, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_BGR666 */ + { + .name = "4:2:2, packed, YUYV", + .fourcc = V4L2_PIX_FMT_YUYV, + .depth = 16, + .flags = 0, + }, + { + .name = "4:2:2, packed, UYVY", + .fourcc = V4L2_PIX_FMT_UYVY, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YVYU + { + .name = "4:2:2, packed YVYU", + .fourcc = V4L2_PIX_FMT_YVYU, + .depth = 16, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_VYUY + { + .name = "4:2:2, packed VYUY", + .fourcc = V4L2_PIX_FMT_VYUY, + .depth = 16, + .flags = 0, + }, +#endif + { + .name = "4:2:2, packed YYUV", + .fourcc = V4L2_PIX_FMT_YYUV, + .depth = 16, + .flags = 0, + }, + { + .name = "YUV-8-8-8-8", + .fourcc = V4L2_PIX_FMT_YUV32, + .depth = 32, + .flags = 0, + }, + { + .name = "8 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_GREY, + .depth = 8, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_Y4 + { + .name = "4 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y4, + .depth = 4, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y4 */ +#ifdef V4L2_PIX_FMT_Y6 + { + .name = "6 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y6, + .depth = 6, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y6 */ +#ifdef V4L2_PIX_FMT_Y10 + { + .name = "10 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y10, + .depth = 10, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y10 */ +#ifdef V4L2_PIX_FMT_Y12 + { + .name = "12 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y12, + .depth = 12, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y12 */ + { + .name = "16 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_Y16, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YUV444 + { + .name = "16 bpp xxxxyyyy uuuuvvvv", + .fourcc = V4L2_PIX_FMT_YUV444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV444 */ +#ifdef V4L2_PIX_FMT_YUV555 + { + .name = "16 bpp YUV-5-5-5", + .fourcc = V4L2_PIX_FMT_YUV555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV555 */ +#ifdef V4L2_PIX_FMT_YUV565 + { + .name = "16 bpp YUV-5-6-5", + .fourcc = V4L2_PIX_FMT_YUV565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV565 */ + +/* bayer formats */ +#ifdef V4L2_PIX_FMT_SRGGB8 + { + .name = "Bayer RGGB 8bit", + .fourcc = V4L2_PIX_FMT_SRGGB8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SRGGB8 */ +#ifdef V4L2_PIX_FMT_SGRBG8 + { + .name = "Bayer GRBG 8bit", + .fourcc = V4L2_PIX_FMT_SGRBG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGRBG8 */ +#ifdef V4L2_PIX_FMT_SGBRG8 + { + .name = "Bayer GBRG 8bit", + .fourcc = V4L2_PIX_FMT_SGBRG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGBRG8 */ +#ifdef V4L2_PIX_FMT_SBGGR8 + { + .name = "Bayer BA81 8bit", + .fourcc = V4L2_PIX_FMT_SBGGR8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SBGGR8 */ + + /* here come the planar formats */ + { + .name = "4:1:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:1:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#ifdef V4L2_PIX_FMT_YUV422P + { + .name = "16 bpp YVU422 planar", + .fourcc = V4L2_PIX_FMT_YUV422P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV422P */ +#ifdef V4L2_PIX_FMT_YUV411P + { + .name = "16 bpp YVU411 planar", + .fourcc = V4L2_PIX_FMT_YUV411P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV411P */ +#ifdef V4L2_PIX_FMT_Y41P + { + .name = "12 bpp YUV 4:1:1", + .fourcc = V4L2_PIX_FMT_Y41P, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_Y41P */ +#ifdef V4L2_PIX_FMT_NV12 + { + .name = "12 bpp Y/CbCr 4:2:0 ", + .fourcc = V4L2_PIX_FMT_NV12, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_NV12 */ + +/* here come the compressed formats */ + +#ifdef V4L2_PIX_FMT_MJPEG + { + .name = "Motion-JPEG", + .fourcc = V4L2_PIX_FMT_MJPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MJPEG */ +#ifdef V4L2_PIX_FMT_JPEG + { + .name = "JFIF JPEG", + .fourcc = V4L2_PIX_FMT_JPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_JPEG */ +#ifdef V4L2_PIX_FMT_DV + { + .name = "DV1394", + .fourcc = V4L2_PIX_FMT_DV, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_DV */ +#ifdef V4L2_PIX_FMT_MPEG + { + .name = "MPEG-1/2/4 Multiplexed", + .fourcc = V4L2_PIX_FMT_MPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG */ +#ifdef V4L2_PIX_FMT_H264 + { + .name = "H264 with start codes", + .fourcc = V4L2_PIX_FMT_H264, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264 */ +#ifdef V4L2_PIX_FMT_H264_NO_SC + { + .name = "H264 without start codes", + .fourcc = V4L2_PIX_FMT_H264_NO_SC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_NO_SC */ +#ifdef V4L2_PIX_FMT_H264_MVC + { + .name = "H264 MVC", + .fourcc = V4L2_PIX_FMT_H264_MVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_MVC */ +#ifdef V4L2_PIX_FMT_H263 + { + .name = "H263", + .fourcc = V4L2_PIX_FMT_H263, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H263 */ +#ifdef V4L2_PIX_FMT_MPEG1 + { + .name = "MPEG-1 ES", + .fourcc = V4L2_PIX_FMT_MPEG1, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG1 */ +#ifdef V4L2_PIX_FMT_MPEG2 + { + .name = "MPEG-2 ES", + .fourcc = V4L2_PIX_FMT_MPEG2, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG2 */ +#ifdef V4L2_PIX_FMT_MPEG4 + { + .name = "MPEG-4 part 2 ES", + .fourcc = V4L2_PIX_FMT_MPEG4, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG4 */ +#ifdef V4L2_PIX_FMT_XVID + { + .name = "Xvid", + .fourcc = V4L2_PIX_FMT_XVID, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_XVID */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_G + { + .name = "SMPTE 421M Annex G compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_G, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_G */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_L + { + .name = "SMPTE 421M Annex L compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_L, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_L */ +#ifdef V4L2_PIX_FMT_VP8 + { + .name = "VP8", + .fourcc = V4L2_PIX_FMT_VP8, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP8 */ +#ifdef V4L2_PIX_FMT_VP9 + { + .name = "VP9", + .fourcc = V4L2_PIX_FMT_VP9, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP9 */ +#ifdef V4L2_PIX_FMT_HEVC + { + .name = "HEVC", + .fourcc = V4L2_PIX_FMT_HEVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_HEVC */ +}; diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 038ccbd9e3ba..de5e4f5145af 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -1,4 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 +ifdef CONFIG_X86_64 +ifdef CONFIG_SATA_AHCI +obj-y += intel-nvme-remap.o +endif +endif + obj-$(CONFIG_PCIE_CADENCE) += cadence/ obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o obj-$(CONFIG_PCI_IXP4XX) += pci-ixp4xx.o diff --git a/drivers/pci/controller/intel-nvme-remap.c b/drivers/pci/controller/intel-nvme-remap.c new file mode 100644 index 000000000000..e105e6f5cc91 --- /dev/null +++ b/drivers/pci/controller/intel-nvme-remap.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Intel remapped NVMe device support. + * + * Copyright (c) 2019 Endless Mobile, Inc. + * Author: Daniel Drake + * + * Some products ship by default with the SATA controller in "RAID" or + * "Intel RST Premium With Intel Optane System Acceleration" mode. Under this + * mode, which we refer to as "remapped NVMe" mode, any installed NVMe + * devices disappear from the PCI bus, and instead their I/O memory becomes + * available within the AHCI device BARs. + * + * This scheme is understood to be a way of avoiding usage of the standard + * Windows NVMe driver under that OS, instead mandating usage of Intel's + * driver instead, which has better power management, and presumably offers + * some RAID/disk-caching solutions too. + * + * Here in this driver, we support the remapped NVMe mode by claiming the + * AHCI device and creating a fake PCIe root port. On the new bus, the + * original AHCI device is exposed with only minor tweaks. Then, fake PCI + * devices corresponding to the remapped NVMe devices are created. The usual + * ahci and nvme drivers are then expected to bind to these devices and + * operate as normal. + * + * The PCI configuration space for the NVMe devices is completely + * unavailable, so we fake a minimal one and hope for the best. + * + * Interrupts are shared between the AHCI and NVMe devices. For simplicity, + * we only support the legacy interrupt here, although MSI support + * could potentially be added later. + */ + +#define MODULE_NAME "intel-nvme-remap" + +#include +#include +#include +#include +#include + +#define AHCI_PCI_BAR_STANDARD 5 + +struct nvme_remap_dev { + struct pci_dev *dev; /* AHCI device */ + struct pci_bus *bus; /* our fake PCI bus */ + struct pci_sysdata sysdata; + int irq_base; /* our fake interrupts */ + + /* + * When we detect an all-ones write to a BAR register, this flag + * is set, so that we return the BAR size on the next read (a + * standard PCI behaviour). + * This includes the assumption that an all-ones BAR write is + * immediately followed by a read of the same register. + */ + bool bar_sizing; + + /* + * Resources copied from the AHCI device, to be regarded as + * resources on our fake bus. + */ + struct resource ahci_resources[PCI_NUM_RESOURCES]; + + /* Resources corresponding to the NVMe devices. */ + struct resource remapped_dev_mem[AHCI_MAX_REMAP]; + + /* Number of remapped NVMe devices found. */ + int num_remapped_devices; +}; + +static inline struct nvme_remap_dev *nrdev_from_bus(struct pci_bus *bus) +{ + return container_of(bus->sysdata, struct nvme_remap_dev, sysdata); +} + + +/******** PCI configuration space **********/ + +/* + * Helper macros for tweaking returned contents of PCI configuration space. + * + * value contains len bytes of data read from reg. + * If fixup_reg is included in that range, fix up the contents of that + * register to fixed_value. + */ +#define NR_FIX8(fixup_reg, fixed_value) do { \ + if (reg <= fixup_reg && fixup_reg < reg + len) \ + ((u8 *) value)[fixup_reg - reg] = (u8) (fixed_value); \ + } while (0) + +#define NR_FIX16(fixup_reg, fixed_value) do { \ + NR_FIX8(fixup_reg, fixed_value); \ + NR_FIX8(fixup_reg + 1, fixed_value >> 8); \ + } while (0) + +#define NR_FIX24(fixup_reg, fixed_value) do { \ + NR_FIX8(fixup_reg, fixed_value); \ + NR_FIX8(fixup_reg + 1, fixed_value >> 8); \ + NR_FIX8(fixup_reg + 2, fixed_value >> 16); \ + } while (0) + +#define NR_FIX32(fixup_reg, fixed_value) do { \ + NR_FIX16(fixup_reg, (u16) fixed_value); \ + NR_FIX16(fixup_reg + 2, fixed_value >> 16); \ + } while (0) + +/* + * Read PCI config space of the slot 0 (AHCI) device. + * We pass through the read request to the underlying device, but + * tweak the results in some cases. + */ +static int nvme_remap_pci_read_slot0(struct pci_bus *bus, int reg, + int len, u32 *value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct pci_bus *ahci_dev_bus = nrdev->dev->bus; + int ret; + + ret = ahci_dev_bus->ops->read(ahci_dev_bus, nrdev->dev->devfn, + reg, len, value); + if (ret) + return ret; + + /* + * Adjust the device class, to prevent this driver from attempting to + * additionally probe the device we're simulating here. + */ + NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_SATA_AHCI); + + /* + * Unset interrupt pin, otherwise ACPI tries to find routing + * info for our virtual IRQ, fails, and complains. + */ + NR_FIX8(PCI_INTERRUPT_PIN, 0); + + /* + * Truncate the AHCI BAR to not include the region that covers the + * hidden devices. This will cause the ahci driver to successfully + * probe th new device (instead of handing it over to this driver). + */ + if (nrdev->bar_sizing) { + NR_FIX32(PCI_BASE_ADDRESS_5, ~(SZ_16K - 1)); + nrdev->bar_sizing = false; + } + + return PCIBIOS_SUCCESSFUL; +} + +/* + * Read PCI config space of a remapped device. + * Since the original PCI config space is inaccessible, we provide a minimal, + * fake config space instead. + */ +static int nvme_remap_pci_read_remapped(struct pci_bus *bus, unsigned int port, + int reg, int len, u32 *value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct resource *remapped_mem; + + if (port > nrdev->num_remapped_devices) + return PCIBIOS_DEVICE_NOT_FOUND; + + *value = 0; + remapped_mem = &nrdev->remapped_dev_mem[port - 1]; + + /* Set a Vendor ID, otherwise Linux assumes no device is present */ + NR_FIX16(PCI_VENDOR_ID, PCI_VENDOR_ID_INTEL); + + /* Always appear on & bus mastering */ + NR_FIX16(PCI_COMMAND, PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + + /* Set class so that nvme driver probes us */ + NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_EXPRESS); + + if (nrdev->bar_sizing) { + NR_FIX32(PCI_BASE_ADDRESS_0, + ~(resource_size(remapped_mem) - 1)); + nrdev->bar_sizing = false; + } else { + resource_size_t mem_start = remapped_mem->start; + + mem_start |= PCI_BASE_ADDRESS_MEM_TYPE_64; + NR_FIX32(PCI_BASE_ADDRESS_0, mem_start); + mem_start >>= 32; + NR_FIX32(PCI_BASE_ADDRESS_1, mem_start); + } + + return PCIBIOS_SUCCESSFUL; +} + +/* Read PCI configuration space. */ +static int nvme_remap_pci_read(struct pci_bus *bus, unsigned int devfn, + int reg, int len, u32 *value) +{ + if (PCI_SLOT(devfn) == 0) + return nvme_remap_pci_read_slot0(bus, reg, len, value); + else + return nvme_remap_pci_read_remapped(bus, PCI_SLOT(devfn), + reg, len, value); +} + +/* + * Write PCI config space of the slot 0 (AHCI) device. + * Apart from the special case of BAR sizing, we disable all writes. + * Otherwise, the ahci driver could make changes (e.g. unset PCI bus master) + * that would affect the operation of the NVMe devices. + */ +static int nvme_remap_pci_write_slot0(struct pci_bus *bus, int reg, + int len, u32 value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct pci_bus *ahci_dev_bus = nrdev->dev->bus; + + if (reg >= PCI_BASE_ADDRESS_0 && reg <= PCI_BASE_ADDRESS_5) { + /* + * Writing all-ones to a BAR means that the size of the + * memory region is being checked. Flag this so that we can + * reply with an appropriate size on the next read. + */ + if (value == ~0) + nrdev->bar_sizing = true; + + return ahci_dev_bus->ops->write(ahci_dev_bus, + nrdev->dev->devfn, + reg, len, value); + } + + return PCIBIOS_SET_FAILED; +} + +/* + * Write PCI config space of a remapped device. + * Since the original PCI config space is inaccessible, we reject all + * writes, except for the special case of BAR probing. + */ +static int nvme_remap_pci_write_remapped(struct pci_bus *bus, + unsigned int port, + int reg, int len, u32 value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + + if (port > nrdev->num_remapped_devices) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* + * Writing all-ones to a BAR means that the size of the memory + * region is being checked. Flag this so that we can reply with + * an appropriate size on the next read. + */ + if (value == ~0 && reg >= PCI_BASE_ADDRESS_0 + && reg <= PCI_BASE_ADDRESS_5) { + nrdev->bar_sizing = true; + return PCIBIOS_SUCCESSFUL; + } + + return PCIBIOS_SET_FAILED; +} + +/* Write PCI configuration space. */ +static int nvme_remap_pci_write(struct pci_bus *bus, unsigned int devfn, + int reg, int len, u32 value) +{ + if (PCI_SLOT(devfn) == 0) + return nvme_remap_pci_write_slot0(bus, reg, len, value); + else + return nvme_remap_pci_write_remapped(bus, PCI_SLOT(devfn), + reg, len, value); +} + +static struct pci_ops nvme_remap_pci_ops = { + .read = nvme_remap_pci_read, + .write = nvme_remap_pci_write, +}; + + +/******** Initialization & exit **********/ + +/* + * Find a PCI domain ID to use for our fake bus. + * Start at 0x10000 to not clash with ACPI _SEG domains (16 bits). + */ +static int find_free_domain(void) +{ + int domain = 0xffff; + struct pci_bus *bus = NULL; + + while ((bus = pci_find_next_bus(bus)) != NULL) + domain = max_t(int, domain, pci_domain_nr(bus)); + + return domain + 1; +} + +static int find_remapped_devices(struct nvme_remap_dev *nrdev, + struct list_head *resources) +{ + void __iomem *mmio; + int i, count = 0; + u32 cap; + + mmio = pcim_iomap(nrdev->dev, AHCI_PCI_BAR_STANDARD, + pci_resource_len(nrdev->dev, + AHCI_PCI_BAR_STANDARD)); + if (!mmio) + return -ENODEV; + + /* Check if this device might have remapped nvme devices. */ + if (pci_resource_len(nrdev->dev, AHCI_PCI_BAR_STANDARD) < SZ_512K || + !(readl(mmio + AHCI_VSCAP) & 1)) + return -ENODEV; + + cap = readq(mmio + AHCI_REMAP_CAP); + for (i = AHCI_MAX_REMAP-1; i >= 0; i--) { + struct resource *remapped_mem; + + if ((cap & (1 << i)) == 0) + continue; + if (readl(mmio + ahci_remap_dcc(i)) + != PCI_CLASS_STORAGE_EXPRESS) + continue; + + /* We've found a remapped device */ + remapped_mem = &nrdev->remapped_dev_mem[count++]; + remapped_mem->start = + pci_resource_start(nrdev->dev, AHCI_PCI_BAR_STANDARD) + + ahci_remap_base(i); + remapped_mem->end = remapped_mem->start + + AHCI_REMAP_N_SIZE - 1; + remapped_mem->flags = IORESOURCE_MEM | IORESOURCE_PCI_FIXED; + pci_add_resource(resources, remapped_mem); + } + + pcim_iounmap(nrdev->dev, mmio); + + if (count == 0) + return -ENODEV; + + nrdev->num_remapped_devices = count; + dev_info(&nrdev->dev->dev, "Found %d remapped NVMe devices\n", + nrdev->num_remapped_devices); + return 0; +} + +static void nvme_remap_remove_root_bus(void *data) +{ + struct pci_bus *bus = data; + + pci_stop_root_bus(bus); + pci_remove_root_bus(bus); +} + +static int nvme_remap_probe(struct pci_dev *dev, + const struct pci_device_id *id) +{ + struct nvme_remap_dev *nrdev; + LIST_HEAD(resources); + int i; + int ret; + struct pci_dev *child; + + nrdev = devm_kzalloc(&dev->dev, sizeof(*nrdev), GFP_KERNEL); + nrdev->sysdata.domain = find_free_domain(); + nrdev->sysdata.nvme_remap_dev = dev; + nrdev->dev = dev; + pci_set_drvdata(dev, nrdev); + + ret = pcim_enable_device(dev); + if (ret < 0) + return ret; + + pci_set_master(dev); + + ret = find_remapped_devices(nrdev, &resources); + if (ret) + return ret; + + /* Add resources from the original AHCI device */ + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = &dev->resource[i]; + + if (res->start) { + struct resource *nr_res = &nrdev->ahci_resources[i]; + + nr_res->start = res->start; + nr_res->end = res->end; + nr_res->flags = res->flags; + pci_add_resource(&resources, nr_res); + } + } + + /* Create virtual interrupts */ + nrdev->irq_base = devm_irq_alloc_descs(&dev->dev, -1, 0, + nrdev->num_remapped_devices + 1, + 0); + if (nrdev->irq_base < 0) + return nrdev->irq_base; + + /* Create and populate PCI bus */ + nrdev->bus = pci_create_root_bus(&dev->dev, 0, &nvme_remap_pci_ops, + &nrdev->sysdata, &resources); + if (!nrdev->bus) + return -ENODEV; + + if (devm_add_action_or_reset(&dev->dev, nvme_remap_remove_root_bus, + nrdev->bus)) + return -ENOMEM; + + /* We don't support sharing MSI interrupts between these devices */ + nrdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI; + + pci_scan_child_bus(nrdev->bus); + + list_for_each_entry(child, &nrdev->bus->devices, bus_list) { + /* + * Prevent PCI core from trying to move memory BARs around. + * The hidden NVMe devices are at fixed locations. + */ + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = &child->resource[i]; + + if (res->flags & IORESOURCE_MEM) + res->flags |= IORESOURCE_PCI_FIXED; + } + + /* Share the legacy IRQ between all devices */ + child->irq = dev->irq; + } + + pci_assign_unassigned_bus_resources(nrdev->bus); + pci_bus_add_devices(nrdev->bus); + + return 0; +} + +static const struct pci_device_id nvme_remap_ids[] = { + /* + * Match all Intel RAID controllers. + * + * There's overlap here with the set of devices detected by the ahci + * driver, but ahci will only successfully probe when there + * *aren't* any remapped NVMe devices, and this driver will only + * successfully probe when there *are* remapped NVMe devices that + * need handling. + */ + { + PCI_VDEVICE(INTEL, PCI_ANY_ID), + .class = PCI_CLASS_STORAGE_RAID << 8, + .class_mask = 0xffffff00, + }, + {0,} +}; +MODULE_DEVICE_TABLE(pci, nvme_remap_ids); + +static struct pci_driver nvme_remap_drv = { + .name = MODULE_NAME, + .id_table = nvme_remap_ids, + .probe = nvme_remap_probe, +}; +module_pci_driver(nvme_remap_drv); + +MODULE_AUTHOR("Daniel Drake "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index db6e142b082d..4c7d236cd069 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3745,6 +3745,106 @@ static void quirk_no_bus_reset(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } +static bool acs_on_downstream; +static bool acs_on_multifunction; + +#define NUM_ACS_IDS 16 +struct acs_on_id { + unsigned short vendor; + unsigned short device; +}; +static struct acs_on_id acs_on_ids[NUM_ACS_IDS]; +static u8 max_acs_id; + +static __init int pcie_acs_override_setup(char *p) +{ + if (!p) + return -EINVAL; + + while (*p) { + if (!strncmp(p, "downstream", 10)) + acs_on_downstream = true; + if (!strncmp(p, "multifunction", 13)) + acs_on_multifunction = true; + if (!strncmp(p, "id:", 3)) { + char opt[5]; + int ret; + long val; + + if (max_acs_id >= NUM_ACS_IDS - 1) { + pr_warn("Out of PCIe ACS override slots (%d)\n", + NUM_ACS_IDS); + goto next; + } + + p += 3; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].vendor = val; + + p += strcspn(p, ":"); + if (*p != ':') { + pr_warn("PCIe ACS invalid ID\n"); + goto next; + } + + p++; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].device = val; + max_acs_id++; + } +next: + p += strcspn(p, ","); + if (*p == ',') + p++; + } + + if (acs_on_downstream || acs_on_multifunction || max_acs_id) + pr_warn("Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA\n"); + + return 0; +} +early_param("pcie_acs_override", pcie_acs_override_setup); + +static int pcie_acs_overrides(struct pci_dev *dev, u16 acs_flags) +{ + int i; + + /* Never override ACS for legacy devices or devices with ACS caps */ + if (!pci_is_pcie(dev) || + pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS)) + return -ENOTTY; + + for (i = 0; i < max_acs_id; i++) + if (acs_on_ids[i].vendor == dev->vendor && + acs_on_ids[i].device == dev->device) + return 1; + + switch (pci_pcie_type(dev)) { + case PCI_EXP_TYPE_DOWNSTREAM: + case PCI_EXP_TYPE_ROOT_PORT: + if (acs_on_downstream) + return 1; + break; + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_UPSTREAM: + case PCI_EXP_TYPE_LEG_END: + case PCI_EXP_TYPE_RC_END: + if (acs_on_multifunction && dev->multifunction) + return 1; + } + + return -ENOTTY; +} /* * Some NVIDIA GPU devices do not work with bus reset, SBR needs to be * prevented for those affected devices. @@ -5192,6 +5292,7 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs }, /* Wangxun nics */ { PCI_VENDOR_ID_WANGXUN, PCI_ANY_ID, pci_quirk_wangxun_nic_acs }, + { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides }, { 0 } }; diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 5522310bab8d..9e1c4634eb7b 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -1524,4 +1524,6 @@ endif # SCSI_LOWLEVEL source "drivers/scsi/device_handler/Kconfig" +source "drivers/scsi/vhba/Kconfig" + endmenu diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index 16de3e41f94c..4e88f6e3e67b 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -152,6 +152,7 @@ obj-$(CONFIG_CHR_DEV_SCH) += ch.o obj-$(CONFIG_SCSI_ENCLOSURE) += ses.o obj-$(CONFIG_SCSI_HISI_SAS) += hisi_sas/ +obj-$(CONFIG_VHBA) += vhba/ # This goes last, so that "real" scsi devices probe earlier obj-$(CONFIG_SCSI_DEBUG) += scsi_debug.o diff --git a/drivers/scsi/vhba/Kconfig b/drivers/scsi/vhba/Kconfig new file mode 100644 index 000000000000..e70a381fe3df --- /dev/null +++ b/drivers/scsi/vhba/Kconfig @@ -0,0 +1,9 @@ +config VHBA + tristate "Virtual (SCSI) Host Bus Adapter" + depends on SCSI + help + This is the in-kernel part of CDEmu, a CD/DVD-ROM device + emulator. + + This driver can also be built as a module. If so, the module + will be called vhba. diff --git a/drivers/scsi/vhba/Makefile b/drivers/scsi/vhba/Makefile new file mode 100644 index 000000000000..2d7524b66199 --- /dev/null +++ b/drivers/scsi/vhba/Makefile @@ -0,0 +1,4 @@ +VHBA_VERSION := 20240917 + +obj-$(CONFIG_VHBA) += vhba.o +ccflags-y := -DVHBA_VERSION=\"$(VHBA_VERSION)\" -Werror diff --git a/drivers/scsi/vhba/vhba.c b/drivers/scsi/vhba/vhba.c new file mode 100644 index 000000000000..878a3be0ba2b --- /dev/null +++ b/drivers/scsi/vhba/vhba.c @@ -0,0 +1,1132 @@ +/* + * vhba.c + * + * Copyright (C) 2007-2012 Chia-I Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define pr_fmt(fmt) "vhba: " fmt + +#include + +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#ifdef CONFIG_COMPAT +#include +#endif +#include +#include +#include +#include +#include +#include + + +MODULE_AUTHOR("Chia-I Wu"); +MODULE_VERSION(VHBA_VERSION); +MODULE_DESCRIPTION("Virtual SCSI HBA"); +MODULE_LICENSE("GPL"); + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#define sdev_dbg(sdev, fmt, a...) \ + dev_dbg(&(sdev)->sdev_gendev, fmt, ##a) +#define scmd_dbg(scmd, fmt, a...) \ + dev_dbg(&(scmd)->device->sdev_gendev, fmt, ##a) +#endif + +#define VHBA_MAX_SECTORS_PER_IO 256 +#define VHBA_MAX_BUS 16 +#define VHBA_MAX_ID 16 +#define VHBA_MAX_DEVICES (VHBA_MAX_BUS * (VHBA_MAX_ID-1)) +#define VHBA_KBUF_SIZE PAGE_SIZE + +#define DATA_TO_DEVICE(dir) ((dir) == DMA_TO_DEVICE || (dir) == DMA_BIDIRECTIONAL) +#define DATA_FROM_DEVICE(dir) ((dir) == DMA_FROM_DEVICE || (dir) == DMA_BIDIRECTIONAL) + + +static int vhba_can_queue = 32; +module_param_named(can_queue, vhba_can_queue, int, 0); + + +enum vhba_req_state { + VHBA_REQ_FREE, + VHBA_REQ_PENDING, + VHBA_REQ_READING, + VHBA_REQ_SENT, + VHBA_REQ_WRITING, +}; + +struct vhba_command { + struct scsi_cmnd *cmd; + /* metatags are per-host. not to be confused with + queue tags that are usually per-lun */ + unsigned long metatag; + int status; + struct list_head entry; +}; + +struct vhba_device { + unsigned int num; + spinlock_t cmd_lock; + struct list_head cmd_list; + wait_queue_head_t cmd_wq; + atomic_t refcnt; + + unsigned char *kbuf; + size_t kbuf_size; +}; + +struct vhba_host { + struct Scsi_Host *shost; + spinlock_t cmd_lock; + int cmd_next; + struct vhba_command *commands; + spinlock_t dev_lock; + struct vhba_device *devices[VHBA_MAX_DEVICES]; + int num_devices; + DECLARE_BITMAP(chgmap, VHBA_MAX_DEVICES); + int chgtype[VHBA_MAX_DEVICES]; + struct work_struct scan_devices; +}; + +#define MAX_COMMAND_SIZE 16 + +struct vhba_request { + __u32 metatag; + __u32 lun; + __u8 cdb[MAX_COMMAND_SIZE]; + __u8 cdb_len; + __u32 data_len; +}; + +struct vhba_response { + __u32 metatag; + __u32 status; + __u32 data_len; +}; + + + +static struct vhba_command *vhba_alloc_command (void); +static void vhba_free_command (struct vhba_command *vcmd); + +static struct platform_device vhba_platform_device; + + + +/* These functions define a symmetric 1:1 mapping between device numbers and + the bus and id. We have reserved the last id per bus for the host itself. */ +static void devnum_to_bus_and_id(unsigned int devnum, unsigned int *bus, unsigned int *id) +{ + *bus = devnum / (VHBA_MAX_ID-1); + *id = devnum % (VHBA_MAX_ID-1); +} + +static unsigned int bus_and_id_to_devnum(unsigned int bus, unsigned int id) +{ + return (bus * (VHBA_MAX_ID-1)) + id; +} + +static struct vhba_device *vhba_device_alloc (void) +{ + struct vhba_device *vdev; + + vdev = kzalloc(sizeof(struct vhba_device), GFP_KERNEL); + if (!vdev) { + return NULL; + } + + spin_lock_init(&vdev->cmd_lock); + INIT_LIST_HEAD(&vdev->cmd_list); + init_waitqueue_head(&vdev->cmd_wq); + atomic_set(&vdev->refcnt, 1); + + vdev->kbuf = NULL; + vdev->kbuf_size = 0; + + return vdev; +} + +static void vhba_device_put (struct vhba_device *vdev) +{ + if (atomic_dec_and_test(&vdev->refcnt)) { + kfree(vdev); + } +} + +static struct vhba_device *vhba_device_get (struct vhba_device *vdev) +{ + atomic_inc(&vdev->refcnt); + + return vdev; +} + +static int vhba_device_queue (struct vhba_device *vdev, struct scsi_cmnd *cmd) +{ + struct vhba_host *vhost; + struct vhba_command *vcmd; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + vcmd = vhba_alloc_command(); + if (!vcmd) { + return SCSI_MLQUEUE_HOST_BUSY; + } + + vcmd->cmd = cmd; + + spin_lock_irqsave(&vdev->cmd_lock, flags); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + vcmd->metatag = scsi_cmd_to_rq(vcmd->cmd)->tag; +#else + vcmd->metatag = vcmd->cmd->request->tag; +#endif + list_add_tail(&vcmd->entry, &vdev->cmd_list); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + wake_up_interruptible(&vdev->cmd_wq); + + return 0; +} + +static int vhba_device_dequeue (struct vhba_device *vdev, struct scsi_cmnd *cmd) +{ + struct vhba_command *vcmd; + int retval; + unsigned long flags; + + spin_lock_irqsave(&vdev->cmd_lock, flags); + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->cmd == cmd) { + list_del_init(&vcmd->entry); + break; + } + } + + /* command not found */ + if (&vcmd->entry == &vdev->cmd_list) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + return SUCCESS; + } + + while (vcmd->status == VHBA_REQ_READING || vcmd->status == VHBA_REQ_WRITING) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + scmd_dbg(cmd, "wait for I/O before aborting\n"); + schedule_timeout(1); + spin_lock_irqsave(&vdev->cmd_lock, flags); + } + + retval = (vcmd->status == VHBA_REQ_SENT) ? FAILED : SUCCESS; + + vhba_free_command(vcmd); + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return retval; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) +static int vhba_slave_alloc(struct scsi_device *sdev) +{ + struct Scsi_Host *shost = sdev->host; + + sdev_dbg(sdev, "enabling tagging (queue depth: %i).\n", sdev->queue_depth); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) + if (!shost_use_blk_mq(shost) && shost->bqt) { +#else + if (shost->bqt) { +#endif + blk_queue_init_tags(sdev->request_queue, sdev->queue_depth, shost->bqt); + } + scsi_adjust_queue_depth(sdev, 0, sdev->queue_depth); + + return 0; +} +#endif + +static void vhba_scan_devices_add (struct vhba_host *vhost, int bus, int id) +{ + struct scsi_device *sdev; + + sdev = scsi_device_lookup(vhost->shost, bus, id, 0); + if (!sdev) { + scsi_add_device(vhost->shost, bus, id, 0); + } else { + dev_warn(&vhost->shost->shost_gendev, "tried to add an already-existing device %d:%d:0!\n", bus, id); + scsi_device_put(sdev); + } +} + +static void vhba_scan_devices_remove (struct vhba_host *vhost, int bus, int id) +{ + struct scsi_device *sdev; + + sdev = scsi_device_lookup(vhost->shost, bus, id, 0); + if (sdev) { + scsi_remove_device(sdev); + scsi_device_put(sdev); + } else { + dev_warn(&vhost->shost->shost_gendev, "tried to remove non-existing device %d:%d:0!\n", bus, id); + } +} + +static void vhba_scan_devices (struct work_struct *work) +{ + struct vhba_host *vhost = container_of(work, struct vhba_host, scan_devices); + unsigned long flags; + int change, exists; + unsigned int devnum; + unsigned int bus, id; + + for (;;) { + spin_lock_irqsave(&vhost->dev_lock, flags); + + devnum = find_first_bit(vhost->chgmap, VHBA_MAX_DEVICES); + if (devnum >= VHBA_MAX_DEVICES) { + spin_unlock_irqrestore(&vhost->dev_lock, flags); + break; + } + change = vhost->chgtype[devnum]; + exists = vhost->devices[devnum] != NULL; + + vhost->chgtype[devnum] = 0; + clear_bit(devnum, vhost->chgmap); + + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + devnum_to_bus_and_id(devnum, &bus, &id); + + if (change < 0) { + dev_dbg(&vhost->shost->shost_gendev, "trying to remove target %d:%d:0\n", bus, id); + vhba_scan_devices_remove(vhost, bus, id); + } else if (change > 0) { + dev_dbg(&vhost->shost->shost_gendev, "trying to add target %d:%d:0\n", bus, id); + vhba_scan_devices_add(vhost, bus, id); + } else { + /* quick sequence of add/remove or remove/add; we determine + which one it was by checking if device structure exists */ + if (exists) { + /* remove followed by add: remove and (re)add */ + dev_dbg(&vhost->shost->shost_gendev, "trying to (re)add target %d:%d:0\n", bus, id); + vhba_scan_devices_remove(vhost, bus, id); + vhba_scan_devices_add(vhost, bus, id); + } else { + /* add followed by remove: no-op */ + dev_dbg(&vhost->shost->shost_gendev, "no-op for target %d:%d:0\n", bus, id); + } + } + } +} + +static int vhba_add_device (struct vhba_device *vdev) +{ + struct vhba_host *vhost; + unsigned int devnum; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + vhba_device_get(vdev); + + spin_lock_irqsave(&vhost->dev_lock, flags); + if (vhost->num_devices >= VHBA_MAX_DEVICES) { + spin_unlock_irqrestore(&vhost->dev_lock, flags); + vhba_device_put(vdev); + return -EBUSY; + } + + for (devnum = 0; devnum < VHBA_MAX_DEVICES; devnum++) { + if (vhost->devices[devnum] == NULL) { + vdev->num = devnum; + vhost->devices[devnum] = vdev; + vhost->num_devices++; + set_bit(devnum, vhost->chgmap); + vhost->chgtype[devnum]++; + break; + } + } + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + schedule_work(&vhost->scan_devices); + + return 0; +} + +static int vhba_remove_device (struct vhba_device *vdev) +{ + struct vhba_host *vhost; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->dev_lock, flags); + set_bit(vdev->num, vhost->chgmap); + vhost->chgtype[vdev->num]--; + vhost->devices[vdev->num] = NULL; + vhost->num_devices--; + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + vhba_device_put(vdev); + + schedule_work(&vhost->scan_devices); + + return 0; +} + +static struct vhba_device *vhba_lookup_device (int devnum) +{ + struct vhba_host *vhost; + struct vhba_device *vdev = NULL; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + if (likely(devnum < VHBA_MAX_DEVICES)) { + spin_lock_irqsave(&vhost->dev_lock, flags); + vdev = vhost->devices[devnum]; + if (vdev) { + vdev = vhba_device_get(vdev); + } + + spin_unlock_irqrestore(&vhost->dev_lock, flags); + } + + return vdev; +} + +static struct vhba_command *vhba_alloc_command (void) +{ + struct vhba_host *vhost; + struct vhba_command *vcmd; + unsigned long flags; + int i; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->cmd_lock, flags); + + vcmd = vhost->commands + vhost->cmd_next++; + if (vcmd->status != VHBA_REQ_FREE) { + for (i = 0; i < vhba_can_queue; i++) { + vcmd = vhost->commands + i; + + if (vcmd->status == VHBA_REQ_FREE) { + vhost->cmd_next = i + 1; + break; + } + } + + if (i == vhba_can_queue) { + vcmd = NULL; + } + } + + if (vcmd) { + vcmd->status = VHBA_REQ_PENDING; + } + + vhost->cmd_next %= vhba_can_queue; + + spin_unlock_irqrestore(&vhost->cmd_lock, flags); + + return vcmd; +} + +static void vhba_free_command (struct vhba_command *vcmd) +{ + struct vhba_host *vhost; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->cmd_lock, flags); + vcmd->status = VHBA_REQ_FREE; + spin_unlock_irqrestore(&vhost->cmd_lock, flags); +} + +static int vhba_queuecommand (struct Scsi_Host *shost, struct scsi_cmnd *cmd) +{ + struct vhba_device *vdev; + int retval; + unsigned int devnum; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + scmd_dbg(cmd, "queue %p tag %i\n", cmd, scsi_cmd_to_rq(cmd)->tag); +#else + scmd_dbg(cmd, "queue %p tag %i\n", cmd, cmd->request->tag); +#endif + + devnum = bus_and_id_to_devnum(cmd->device->channel, cmd->device->id); + vdev = vhba_lookup_device(devnum); + if (!vdev) { + scmd_dbg(cmd, "no such device\n"); + + cmd->result = DID_NO_CONNECT << 16; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(cmd); +#else + cmd->scsi_done(cmd); +#endif + + return 0; + } + + retval = vhba_device_queue(vdev, cmd); + + vhba_device_put(vdev); + + return retval; +} + +static int vhba_abort (struct scsi_cmnd *cmd) +{ + struct vhba_device *vdev; + int retval = SUCCESS; + unsigned int devnum; + + scmd_dbg(cmd, "abort %p\n", cmd); + + devnum = bus_and_id_to_devnum(cmd->device->channel, cmd->device->id); + vdev = vhba_lookup_device(devnum); + if (vdev) { + retval = vhba_device_dequeue(vdev, cmd); + vhba_device_put(vdev); + } else { + cmd->result = DID_NO_CONNECT << 16; + } + + return retval; +} + +static struct scsi_host_template vhba_template = { + .module = THIS_MODULE, + .name = "vhba", + .proc_name = "vhba", + .queuecommand = vhba_queuecommand, + .eh_abort_handler = vhba_abort, + .this_id = -1, + .max_sectors = VHBA_MAX_SECTORS_PER_IO, + .sg_tablesize = 256, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) + .slave_alloc = vhba_slave_alloc, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(6, 14, 0) + .tag_alloc_policy = BLK_TAG_ALLOC_RR, +#else + .tag_alloc_policy_rr = true, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + .use_blk_tags = 1, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) + .max_segment_size = VHBA_KBUF_SIZE, +#endif +}; + +static ssize_t do_request (struct vhba_device *vdev, unsigned long metatag, struct scsi_cmnd *cmd, char __user *buf, size_t buf_len) +{ + struct vhba_request vreq; + ssize_t ret; + + scmd_dbg(cmd, "request %lu (%p), cdb 0x%x, bufflen %d, sg count %d\n", + metatag, cmd, cmd->cmnd[0], scsi_bufflen(cmd), scsi_sg_count(cmd)); + + ret = sizeof(vreq); + if (DATA_TO_DEVICE(cmd->sc_data_direction)) { + ret += scsi_bufflen(cmd); + } + + if (ret > buf_len) { + scmd_dbg(cmd, "buffer too small (%zd < %zd) for a request\n", buf_len, ret); + return -EIO; + } + + vreq.metatag = metatag; + vreq.lun = cmd->device->lun; + memcpy(vreq.cdb, cmd->cmnd, MAX_COMMAND_SIZE); + vreq.cdb_len = cmd->cmd_len; + vreq.data_len = scsi_bufflen(cmd); + + if (copy_to_user(buf, &vreq, sizeof(vreq))) { + return -EFAULT; + } + + if (DATA_TO_DEVICE(cmd->sc_data_direction) && vreq.data_len) { + buf += sizeof(vreq); + + if (scsi_sg_count(cmd)) { + unsigned char *kaddr, *uaddr; + struct scatterlist *sglist = scsi_sglist(cmd); + struct scatterlist *sg; + int i; + + uaddr = (unsigned char *) buf; + + for_each_sg(sglist, sg, scsi_sg_count(cmd), i) { + size_t len = sg->length; + + if (len > vdev->kbuf_size) { + scmd_dbg(cmd, "segment size (%zu) exceeds kbuf size (%zu)!", len, vdev->kbuf_size); + len = vdev->kbuf_size; + } + + kaddr = kmap_atomic(sg_page(sg)); + memcpy(vdev->kbuf, kaddr + sg->offset, len); + kunmap_atomic(kaddr); + + if (copy_to_user(uaddr, vdev->kbuf, len)) { + return -EFAULT; + } + uaddr += len; + } + } else { + if (copy_to_user(buf, scsi_sglist(cmd), vreq.data_len)) { + return -EFAULT; + } + } + } + + return ret; +} + +static ssize_t do_response (struct vhba_device *vdev, unsigned long metatag, struct scsi_cmnd *cmd, const char __user *buf, size_t buf_len, struct vhba_response *res) +{ + ssize_t ret = 0; + + scmd_dbg(cmd, "response %lu (%p), status %x, data len %d, sg count %d\n", + metatag, cmd, res->status, res->data_len, scsi_sg_count(cmd)); + + if (res->status) { + if (res->data_len > SCSI_SENSE_BUFFERSIZE) { + scmd_dbg(cmd, "truncate sense (%d < %d)", SCSI_SENSE_BUFFERSIZE, res->data_len); + res->data_len = SCSI_SENSE_BUFFERSIZE; + } + + if (copy_from_user(cmd->sense_buffer, buf, res->data_len)) { + return -EFAULT; + } + + cmd->result = res->status; + + ret += res->data_len; + } else if (DATA_FROM_DEVICE(cmd->sc_data_direction) && scsi_bufflen(cmd)) { + size_t to_read; + + if (res->data_len > scsi_bufflen(cmd)) { + scmd_dbg(cmd, "truncate data (%d < %d)\n", scsi_bufflen(cmd), res->data_len); + res->data_len = scsi_bufflen(cmd); + } + + to_read = res->data_len; + + if (scsi_sg_count(cmd)) { + unsigned char *kaddr, *uaddr; + struct scatterlist *sglist = scsi_sglist(cmd); + struct scatterlist *sg; + int i; + + uaddr = (unsigned char *)buf; + + for_each_sg(sglist, sg, scsi_sg_count(cmd), i) { + size_t len = (sg->length < to_read) ? sg->length : to_read; + + if (len > vdev->kbuf_size) { + scmd_dbg(cmd, "segment size (%zu) exceeds kbuf size (%zu)!", len, vdev->kbuf_size); + len = vdev->kbuf_size; + } + + if (copy_from_user(vdev->kbuf, uaddr, len)) { + return -EFAULT; + } + uaddr += len; + + kaddr = kmap_atomic(sg_page(sg)); + memcpy(kaddr + sg->offset, vdev->kbuf, len); + kunmap_atomic(kaddr); + + to_read -= len; + if (to_read == 0) { + break; + } + } + } else { + if (copy_from_user(scsi_sglist(cmd), buf, res->data_len)) { + return -EFAULT; + } + + to_read -= res->data_len; + } + + scsi_set_resid(cmd, to_read); + + ret += res->data_len - to_read; + } + + return ret; +} + +static struct vhba_command *next_command (struct vhba_device *vdev) +{ + struct vhba_command *vcmd; + + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->status == VHBA_REQ_PENDING) { + break; + } + } + + if (&vcmd->entry == &vdev->cmd_list) { + vcmd = NULL; + } + + return vcmd; +} + +static struct vhba_command *match_command (struct vhba_device *vdev, __u32 metatag) +{ + struct vhba_command *vcmd; + + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->metatag == metatag) { + break; + } + } + + if (&vcmd->entry == &vdev->cmd_list) { + vcmd = NULL; + } + + return vcmd; +} + +static struct vhba_command *wait_command (struct vhba_device *vdev, unsigned long flags) +{ + struct vhba_command *vcmd; + DEFINE_WAIT(wait); + + while (!(vcmd = next_command(vdev))) { + if (signal_pending(current)) { + break; + } + + prepare_to_wait(&vdev->cmd_wq, &wait, TASK_INTERRUPTIBLE); + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + schedule(); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + } + + finish_wait(&vdev->cmd_wq, &wait); + if (vcmd) { + vcmd->status = VHBA_REQ_READING; + } + + return vcmd; +} + +static ssize_t vhba_ctl_read (struct file *file, char __user *buf, size_t buf_len, loff_t *offset) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + ssize_t ret; + unsigned long flags; + + vdev = file->private_data; + + /* Get next command */ + if (file->f_flags & O_NONBLOCK) { + /* Non-blocking variant */ + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = next_command(vdev); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + if (!vcmd) { + return -EWOULDBLOCK; + } + } else { + /* Blocking variant */ + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = wait_command(vdev, flags); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + if (!vcmd) { + return -ERESTARTSYS; + } + } + + ret = do_request(vdev, vcmd->metatag, vcmd->cmd, buf, buf_len); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (ret >= 0) { + vcmd->status = VHBA_REQ_SENT; + *offset += ret; + } else { + vcmd->status = VHBA_REQ_PENDING; + } + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return ret; +} + +static ssize_t vhba_ctl_write (struct file *file, const char __user *buf, size_t buf_len, loff_t *offset) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + struct vhba_response res; + ssize_t ret; + unsigned long flags; + + if (buf_len < sizeof(res)) { + return -EIO; + } + + if (copy_from_user(&res, buf, sizeof(res))) { + return -EFAULT; + } + + vdev = file->private_data; + + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = match_command(vdev, res.metatag); + if (!vcmd || vcmd->status != VHBA_REQ_SENT) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + pr_debug("ctl dev #%u not expecting response\n", vdev->num); + return -EIO; + } + vcmd->status = VHBA_REQ_WRITING; + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + ret = do_response(vdev, vcmd->metatag, vcmd->cmd, buf + sizeof(res), buf_len - sizeof(res), &res); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (ret >= 0) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(vcmd->cmd); +#else + vcmd->cmd->scsi_done(vcmd->cmd); +#endif + ret += sizeof(res); + + /* don't compete with vhba_device_dequeue */ + if (!list_empty(&vcmd->entry)) { + list_del_init(&vcmd->entry); + vhba_free_command(vcmd); + } + } else { + vcmd->status = VHBA_REQ_SENT; + } + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return ret; +} + +static long vhba_ctl_ioctl (struct file *file, unsigned int cmd, unsigned long arg) +{ + struct vhba_device *vdev = file->private_data; + struct vhba_host *vhost = platform_get_drvdata(&vhba_platform_device); + + switch (cmd) { + case 0xBEEF001: { + unsigned int ident[4]; /* host, channel, id, lun */ + + ident[0] = vhost->shost->host_no; + devnum_to_bus_and_id(vdev->num, &ident[1], &ident[2]); + ident[3] = 0; /* lun */ + + if (copy_to_user((void *) arg, ident, sizeof(ident))) { + return -EFAULT; + } + + return 0; + } + case 0xBEEF002: { + unsigned int devnum = vdev->num; + + if (copy_to_user((void *) arg, &devnum, sizeof(devnum))) { + return -EFAULT; + } + + return 0; + } + } + + return -ENOTTY; +} + +#ifdef CONFIG_COMPAT +static long vhba_ctl_compat_ioctl (struct file *file, unsigned int cmd, unsigned long arg) +{ + unsigned long compat_arg = (unsigned long)compat_ptr(arg); + return vhba_ctl_ioctl(file, cmd, compat_arg); +} +#endif + +static unsigned int vhba_ctl_poll (struct file *file, poll_table *wait) +{ + struct vhba_device *vdev = file->private_data; + unsigned int mask = 0; + unsigned long flags; + + poll_wait(file, &vdev->cmd_wq, wait); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (next_command(vdev)) { + mask |= POLLIN | POLLRDNORM; + } + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return mask; +} + +static int vhba_ctl_open (struct inode *inode, struct file *file) +{ + struct vhba_device *vdev; + int retval; + + pr_debug("ctl dev open\n"); + + /* check if vhba is probed */ + if (!platform_get_drvdata(&vhba_platform_device)) { + return -ENODEV; + } + + vdev = vhba_device_alloc(); + if (!vdev) { + return -ENOMEM; + } + + vdev->kbuf_size = VHBA_KBUF_SIZE; + vdev->kbuf = kzalloc(vdev->kbuf_size, GFP_KERNEL); + if (!vdev->kbuf) { + return -ENOMEM; + } + + if (!(retval = vhba_add_device(vdev))) { + file->private_data = vdev; + } + + vhba_device_put(vdev); + + return retval; +} + +static int vhba_ctl_release (struct inode *inode, struct file *file) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + unsigned long flags; + + vdev = file->private_data; + + pr_debug("ctl dev release\n"); + + vhba_device_get(vdev); + vhba_remove_device(vdev); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + WARN_ON(vcmd->status == VHBA_REQ_READING || vcmd->status == VHBA_REQ_WRITING); + + scmd_dbg(vcmd->cmd, "device released with command %lu (%p)\n", vcmd->metatag, vcmd->cmd); + vcmd->cmd->result = DID_NO_CONNECT << 16; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(vcmd->cmd); +#else + vcmd->cmd->scsi_done(vcmd->cmd); +#endif + vhba_free_command(vcmd); + } + INIT_LIST_HEAD(&vdev->cmd_list); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + kfree(vdev->kbuf); + vdev->kbuf = NULL; + + vhba_device_put(vdev); + + return 0; +} + +static struct file_operations vhba_ctl_fops = { + .owner = THIS_MODULE, + .open = vhba_ctl_open, + .release = vhba_ctl_release, + .read = vhba_ctl_read, + .write = vhba_ctl_write, + .poll = vhba_ctl_poll, + .unlocked_ioctl = vhba_ctl_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = vhba_ctl_compat_ioctl, +#endif +}; + +static struct miscdevice vhba_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "vhba_ctl", + .fops = &vhba_ctl_fops, +}; + +static int vhba_probe (struct platform_device *pdev) +{ + struct Scsi_Host *shost; + struct vhba_host *vhost; + int i; + + vhba_can_queue = clamp(vhba_can_queue, 1, 256); + + shost = scsi_host_alloc(&vhba_template, sizeof(struct vhba_host)); + if (!shost) { + return -ENOMEM; + } + + shost->max_channel = VHBA_MAX_BUS-1; + shost->max_id = VHBA_MAX_ID; + /* we don't support lun > 0 */ + shost->max_lun = 1; + shost->max_cmd_len = MAX_COMMAND_SIZE; + shost->can_queue = vhba_can_queue; + shost->cmd_per_lun = vhba_can_queue; + + vhost = (struct vhba_host *)shost->hostdata; + memset(vhost, 0, sizeof(struct vhba_host)); + + vhost->shost = shost; + vhost->num_devices = 0; + spin_lock_init(&vhost->dev_lock); + spin_lock_init(&vhost->cmd_lock); + INIT_WORK(&vhost->scan_devices, vhba_scan_devices); + vhost->cmd_next = 0; + vhost->commands = kzalloc(vhba_can_queue * sizeof(struct vhba_command), GFP_KERNEL); + if (!vhost->commands) { + return -ENOMEM; + } + + for (i = 0; i < vhba_can_queue; i++) { + vhost->commands[i].status = VHBA_REQ_FREE; + } + + platform_set_drvdata(pdev, vhost); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + i = scsi_init_shared_tag_map(shost, vhba_can_queue); + if (i) return i; +#endif + + if (scsi_add_host(shost, &pdev->dev)) { + scsi_host_put(shost); + return -ENOMEM; + } + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) +static int vhba_remove (struct platform_device *pdev) +#else +static void vhba_remove (struct platform_device *pdev) +#endif +{ + struct vhba_host *vhost; + struct Scsi_Host *shost; + + vhost = platform_get_drvdata(pdev); + shost = vhost->shost; + + scsi_remove_host(shost); + scsi_host_put(shost); + + kfree(vhost->commands); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + return 0; +#endif +} + +static void vhba_release (struct device * dev) +{ + return; +} + +static struct platform_device vhba_platform_device = { + .name = "vhba", + .id = -1, + .dev = { + .release = vhba_release, + }, +}; + +static struct platform_driver vhba_platform_driver = { + .driver = { + .owner = THIS_MODULE, + .name = "vhba", + }, + .probe = vhba_probe, + .remove = vhba_remove, +}; + +static int __init vhba_init (void) +{ + int ret; + + ret = platform_device_register(&vhba_platform_device); + if (ret < 0) { + return ret; + } + + ret = platform_driver_register(&vhba_platform_driver); + if (ret < 0) { + platform_device_unregister(&vhba_platform_device); + return ret; + } + + ret = misc_register(&vhba_miscdev); + if (ret < 0) { + platform_driver_unregister(&vhba_platform_driver); + platform_device_unregister(&vhba_platform_device); + return ret; + } + + return 0; +} + +static void __exit vhba_exit(void) +{ + misc_deregister(&vhba_miscdev); + platform_driver_unregister(&vhba_platform_driver); + platform_device_unregister(&vhba_platform_device); +} + +module_init(vhba_init); +module_exit(vhba_exit); + diff --git a/include/linux/mm.h b/include/linux/mm.h index fa538feaa8d9..8dd5828e7ccb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -190,10 +190,18 @@ static inline void __mm_zero_struct_page(struct page *page) * that. */ #define MAPCOUNT_ELF_CORE_MARGIN (5) -#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) +#define DEFAULT_MAX_MAP_COUNT (INT_MAX - MAPCOUNT_ELF_CORE_MARGIN) extern int sysctl_max_map_count; +extern bool sysctl_workingset_protection; +extern u8 sysctl_anon_min_ratio; +extern u8 sysctl_clean_low_ratio; +extern u8 sysctl_clean_min_ratio; +int vm_workingset_protection_update_handler( + const struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + extern unsigned long sysctl_user_reserve_kbytes; extern unsigned long sysctl_admin_reserve_kbytes; diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index e63fbfbd5b0f..dca57c6e3157 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1318,7 +1318,7 @@ struct readahead_control { ._index = i, \ } -#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) +#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) void page_cache_ra_unbounded(struct readahead_control *, unsigned long nr_to_read, unsigned long lookahead_count); diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index a0bb6d012137..93129fea552e 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -168,6 +168,8 @@ static inline void set_userns_rlimit_max(struct user_namespace *ns, #ifdef CONFIG_USER_NS +extern int unprivileged_userns_clone; + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { if (ns) @@ -201,6 +203,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); struct ns_common *ns_get_owner(struct ns_common *ns); #else +#define unprivileged_userns_clone 0 + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { return &init_user_ns; diff --git a/include/linux/wait.h b/include/linux/wait.h index 965a19809c7e..3d442267a256 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -163,6 +163,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); +extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); @@ -1195,6 +1196,7 @@ do { \ */ void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); +void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); diff --git a/init/Kconfig b/init/Kconfig index 2e15b4a8478e..6cead1ecd2f5 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -167,6 +167,10 @@ config THREAD_INFO_IN_TASK menu "General setup" +config CACHY + bool "Some kernel tweaks by CachyOS" + default y + config BROKEN bool @@ -1349,6 +1353,22 @@ config USER_NS If unsure, say N. +config USER_NS_UNPRIVILEGED + bool "Allow unprivileged users to create namespaces" + default y + depends on USER_NS + help + When disabled, unprivileged users will not be able to create + new namespaces. Allowing users to create their own namespaces + has been part of several recent local privilege escalation + exploits, so if you need user namespaces but are + paranoid^Wsecurity-conscious you want to disable this. + + This setting can be overridden at runtime via the + kernel.unprivileged_userns_clone sysctl. + + If unsure, say Y. + config PID_NS bool "PID Namespaces" default y @@ -1498,6 +1518,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE with the "-O2" compiler flag for best performance and most helpful compile-time warnings. +config CC_OPTIMIZE_FOR_PERFORMANCE_O3 + bool "Optimize more for performance (-O3)" + help + Choosing this option will pass "-O3" to your compiler to optimize + the kernel yet more for performance. + config CC_OPTIMIZE_FOR_SIZE bool "Optimize for size (-Os)" help diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz index ce1435cb08b1..e1359db5561e 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz @@ -40,6 +40,27 @@ choice on SMP and NUMA systems and exactly dividing by both PAL and NTSC frame rates for video and multimedia work. + config HZ_500 + bool "500 HZ" + help + 500 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_600 + bool "600 HZ" + help + 600 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_750 + bool "750 HZ" + help + 750 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + config HZ_1000 bool "1000 HZ" help @@ -53,6 +74,9 @@ config HZ default 100 if HZ_100 default 250 if HZ_250 default 300 if HZ_300 + default 500 if HZ_500 + default 600 if HZ_600 + default 750 if HZ_750 default 1000 if HZ_1000 config SCHED_HRTICK diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 54ea59ff8fbe..18f87e0dd137 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -88,7 +88,7 @@ endchoice config PREEMPT_RT bool "Fully Preemptible Kernel (Real-Time)" - depends on EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST + depends on ARCH_SUPPORTS_RT && !COMPILE_TEST select PREEMPTION help This option turns the kernel into a real-time kernel by replacing diff --git a/kernel/fork.c b/kernel/fork.c index 1ee8eb11f38b..ea6c2e00a8ba 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -106,6 +106,10 @@ #include #include +#ifdef CONFIG_USER_NS +#include +#endif + #include #include #include @@ -1933,6 +1937,10 @@ __latent_entropy struct task_struct *copy_process( if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) return ERR_PTR(-EINVAL); + if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) + if (!capable(CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. @@ -3099,6 +3107,12 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; + if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { + err = -EPERM; + if (!capable(CAP_SYS_ADMIN)) + goto bad_unshare_out; + } + err = check_unshare_flags(unshare_flags); if (err) goto bad_unshare_out; diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 2ddb827e3bea..464049c4af3f 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) struct task_struct *new, *owner; unsigned long flags, new_flags; enum owner_state state; + int i = 0; lockdep_assert_preemption_disabled(); @@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) break; } - cpu_relax(); + if (i++ > 1000) + cpu_relax(); } return state; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 042ab0863ccc..3ce0e3a56636 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -76,10 +76,19 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; * * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ +#ifdef CONFIG_CACHY +unsigned int sysctl_sched_base_slice = 350000ULL; +static unsigned int normalized_sysctl_sched_base_slice = 350000ULL; +#else unsigned int sysctl_sched_base_slice = 700000ULL; static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; +#endif /* CONFIG_CACHY */ +#ifdef CONFIG_CACHY +__read_mostly unsigned int sysctl_sched_migration_cost = 300000UL; +#else __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; +#endif static int __init setup_sched_thermal_decay_shift(char *str) { @@ -124,8 +133,12 @@ int __weak arch_asym_cpu_priority(int cpu) * * (default: 5 msec, units: microseconds) */ +#ifdef CONFIG_CACHY +static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; +#else static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +#endif #ifdef CONFIG_NUMA_BALANCING /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 83e3aa917142..f2ceeb50d4a2 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2816,7 +2816,7 @@ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); extern void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags); -#ifdef CONFIG_PREEMPT_RT +#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) # define SCHED_NR_MIGRATE_BREAK 8 #else # define SCHED_NR_MIGRATE_BREAK 32 diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 51e38f5f4701..c5cc616484ba 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ } EXPORT_SYMBOL_GPL(add_wait_queue_priority); +void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) +{ + unsigned long flags; + + wq_entry->flags |= WQ_FLAG_EXCLUSIVE; + spin_lock_irqsave(&wq_head->lock, flags); + __add_wait_queue(wq_head, wq_entry); + spin_unlock_irqrestore(&wq_head->lock, flags); +} +EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); + void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) { unsigned long flags; @@ -258,6 +269,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent } EXPORT_SYMBOL(prepare_to_wait_exclusive); +void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) +{ + unsigned long flags; + + wq_entry->flags |= WQ_FLAG_EXCLUSIVE; + spin_lock_irqsave(&wq_head->lock, flags); + if (list_empty(&wq_entry->entry)) + __add_wait_queue(wq_head, wq_entry); + set_current_state(state); + spin_unlock_irqrestore(&wq_head->lock, flags); +} +EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); + void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) { wq_entry->flags = flags; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 9b4f0cff76ea..927ee868db88 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -63,6 +63,9 @@ #ifdef CONFIG_RT_MUTEXES #include #endif +#ifdef CONFIG_USER_NS +#include +#endif /* shared constants to be used in various sysctls */ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; @@ -1581,6 +1584,15 @@ int proc_do_static_key(const struct ctl_table *table, int write, } static const struct ctl_table kern_table[] = { +#ifdef CONFIG_USER_NS + { + .procname = "unprivileged_userns_clone", + .data = &unprivileged_userns_clone, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #ifdef CONFIG_PROC_SYSCTL { .procname = "tainted", diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 682f40d5632d..434a25f7b2ed 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -22,6 +22,13 @@ #include #include +/* sysctl */ +#ifdef CONFIG_USER_NS_UNPRIVILEGED +int unprivileged_userns_clone = 1; +#else +int unprivileged_userns_clone; +#endif + static struct kmem_cache *user_ns_cachep __ro_after_init; static DEFINE_MUTEX(userns_state_mutex); diff --git a/mm/Kconfig b/mm/Kconfig index 781be3240e21..5bd761b2976e 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -462,6 +462,69 @@ config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP config ARCH_WANT_HUGETLB_VMEMMAP_PREINIT bool +config ANON_MIN_RATIO + int "Default value for vm.anon_min_ratio" + depends on SYSCTL + range 0 100 + default 1 + help + This option sets the default value for vm.anon_min_ratio sysctl knob. + + The vm.anon_min_ratio sysctl knob provides *hard* protection of + anonymous pages. The anonymous pages on the current node won't be + reclaimed under any conditions when their amount is below + vm.anon_min_ratio. This knob may be used to prevent excessive swap + thrashing when anonymous memory is low (for example, when memory is + going to be overfilled by compressed data of zram module). + + Setting this value too high (close to MemTotal) can result in + inability to swap and can lead to early OOM under memory pressure. + +config CLEAN_LOW_RATIO + int "Default value for vm.clean_low_ratio" + depends on SYSCTL + range 0 100 + default 15 + help + This option sets the default value for vm.clean_low_ratio sysctl knob. + + The vm.clean_low_ratio sysctl knob provides *best-effort* + protection of clean file pages. The file pages on the current node + won't be reclaimed under memory pressure when the amount of clean file + pages is below vm.clean_low_ratio *unless* we threaten to OOM. + Protection of clean file pages using this knob may be used when + swapping is still possible to + - prevent disk I/O thrashing under memory pressure; + - improve performance in disk cache-bound tasks under memory + pressure. + + Setting it to a high value may result in a early eviction of anonymous + pages into the swap space by attempting to hold the protected amount + of clean file pages in memory. + +config CLEAN_MIN_RATIO + int "Default value for vm.clean_min_ratio" + depends on SYSCTL + range 0 100 + default 4 + help + This option sets the default value for vm.clean_min_ratio sysctl knob. + + The vm.clean_min_ratio sysctl knob provides *hard* protection of + clean file pages. The file pages on the current node won't be + reclaimed under memory pressure when the amount of clean file pages is + below vm.clean_min_ratio. Hard protection of clean file pages using + this knob may be used to + - prevent disk I/O thrashing under memory pressure even with no free + swap space; + - improve performance in disk cache-bound tasks under memory + pressure; + - avoid high latency and prevent livelock in near-OOM conditions. + + Setting it to a high value may result in a early out-of-memory condition + due to the inability to reclaim the protected amount of clean file pages + when other types of pages cannot be reclaimed. + config HAVE_MEMBLOCK_PHYS_MAP bool @@ -658,7 +721,7 @@ config COMPACTION config COMPACT_UNEVICTABLE_DEFAULT int depends on COMPACTION - default 0 if PREEMPT_RT + default 0 if PREEMPT_RT || CACHY default 1 # diff --git a/mm/compaction.c b/mm/compaction.c index 3925cb61dbb8..42374158dc17 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1923,7 +1923,11 @@ static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNE * aggressively the kernel should compact memory in the * background. It takes values in the range [0, 100]. */ +#ifdef CONFIG_CACHY +static unsigned int __read_mostly sysctl_compaction_proactiveness; +#else static unsigned int __read_mostly sysctl_compaction_proactiveness = 20; +#endif static int sysctl_extfrag_threshold = 500; static int __read_mostly sysctl_compact_memory; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 49b98082c540..788e87fb5969 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -64,7 +64,11 @@ unsigned long transparent_hugepage_flags __read_mostly = #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE (1<> (20 - PAGE_SHIFT); /* Use a smaller cluster for small-memory machines */ @@ -1098,6 +1102,7 @@ void __init swap_setup(void) page_cluster = 2; else page_cluster = 3; +#endif /* CONFIG_CACHY */ /* * Right now other parts of the system means that we * _really_ don't want to cluster much more diff --git a/mm/util.c b/mm/util.c index 0b270c43d7d1..0b4db8bfe8cf 100644 --- a/mm/util.c +++ b/mm/util.c @@ -857,6 +857,40 @@ static const struct ctl_table util_sysctl_table[] = { .mode = 0644, .proc_handler = proc_doulongvec_minmax, }, + { + .procname = "workingset_protection", + .data = &sysctl_workingset_protection, + .maxlen = sizeof(bool), + .mode = 0644, + .proc_handler = &proc_dobool, + }, + { + .procname = "anon_min_ratio", + .data = &sysctl_anon_min_ratio, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = &vm_workingset_protection_update_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "clean_low_ratio", + .data = &sysctl_clean_low_ratio, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = &vm_workingset_protection_update_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "clean_min_ratio", + .data = &sysctl_clean_min_ratio, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = &vm_workingset_protection_update_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, }; static int __init init_vm_util_sysctls(void) diff --git a/mm/vmpressure.c b/mm/vmpressure.c index bd5183dfd879..3a410f53a07c 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; * essence, they are percents: the higher the value, the more number * unsuccessful reclaims there were. */ +#ifdef CONFIG_CACHY +static const unsigned int vmpressure_level_med = 65; +#else static const unsigned int vmpressure_level_med = 60; +#endif static const unsigned int vmpressure_level_critical = 95; /* diff --git a/mm/vmscan.c b/mm/vmscan.c index 424412680cfc..0d58febd4819 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -148,6 +148,15 @@ struct scan_control { /* The file folios on the current node are dangerously low */ unsigned int file_is_tiny:1; + /* The anonymous pages on the current node are below vm.anon_min_ratio */ + unsigned int anon_below_min:1; + + /* The clean file pages on the current node are below vm.clean_low_ratio */ + unsigned int clean_below_low:1; + + /* The clean file pages on the current node are below vm.clean_min_ratio */ + unsigned int clean_below_min:1; + /* Always discard instead of demoting to lower tier memory */ unsigned int no_demotion:1; @@ -197,10 +206,23 @@ struct scan_control { #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) #endif +bool sysctl_workingset_protection __read_mostly = true; +u8 sysctl_anon_min_ratio __read_mostly = CONFIG_ANON_MIN_RATIO; +u8 sysctl_clean_low_ratio __read_mostly = CONFIG_CLEAN_LOW_RATIO; +u8 sysctl_clean_min_ratio __read_mostly = CONFIG_CLEAN_MIN_RATIO; +static u64 sysctl_anon_min_ratio_kb __read_mostly = 0; +static u64 sysctl_clean_low_ratio_kb __read_mostly = 0; +static u64 sysctl_clean_min_ratio_kb __read_mostly = 0; +static u64 workingset_protection_prev_totalram __read_mostly = 0; + /* * From 0 .. MAX_SWAPPINESS. Higher means more swappy. */ +#ifdef CONFIG_CACHY +int vm_swappiness = 100; +#else int vm_swappiness = 60; +#endif #ifdef CONFIG_MEMCG @@ -1165,6 +1187,10 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, if (!sc->may_unmap && folio_mapped(folio)) goto keep_locked; + if (folio_is_file_lru(folio) ? sc->clean_below_min : + (sc->anon_below_min && !sc->clean_below_min)) + goto keep_locked; + /* * The number of dirty pages determines if a node is marked * reclaim_congested. kswapd will stall and start writing @@ -2551,6 +2577,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } + /* + * Force-scan anon if clean file pages is under vm.clean_low_ratio + * or vm.clean_min_ratio. + */ + if (sc->clean_below_low || sc->clean_below_min) { + scan_balance = SCAN_ANON; + goto out; + } + /* * If there is enough inactive page cache, we do not reclaim * anything from the anonymous working right now to make sure @@ -2669,6 +2704,14 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, BUG(); } + /* + * Hard protection of the working set. + * Don't reclaim anon/file pages when the amount is + * below the watermark of the same type. + */ + if (file ? sc->clean_below_min : sc->anon_below_min) + scan = 0; + nr[lru] = scan; } } @@ -2689,6 +2732,96 @@ static bool can_age_anon_pages(struct lruvec *lruvec, lruvec_memcg(lruvec)); } +int vm_workingset_protection_update_handler(const struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write) + return ret; + + workingset_protection_prev_totalram = 0; + + return 0; +} + +static void prepare_workingset_protection(pg_data_t *pgdat, struct scan_control *sc) +{ + unsigned long node_mem_total; + struct sysinfo i; + + if (!(sysctl_workingset_protection)) { + sc->anon_below_min = 0; + sc->clean_below_low = 0; + sc->clean_below_min = 0; + return; + } + + if (likely(sysctl_anon_min_ratio || + sysctl_clean_low_ratio || + sysctl_clean_min_ratio)) { +#ifdef CONFIG_NUMA + si_meminfo_node(&i, pgdat->node_id); +#else //CONFIG_NUMA + si_meminfo(&i); +#endif //CONFIG_NUMA + node_mem_total = i.totalram; + + if (unlikely(workingset_protection_prev_totalram != node_mem_total)) { + sysctl_anon_min_ratio_kb = + node_mem_total * sysctl_anon_min_ratio / 100; + sysctl_clean_low_ratio_kb = + node_mem_total * sysctl_clean_low_ratio / 100; + sysctl_clean_min_ratio_kb = + node_mem_total * sysctl_clean_min_ratio / 100; + workingset_protection_prev_totalram = node_mem_total; + } + } + + /* + * Check the number of anonymous pages to protect them from + * reclaiming if their amount is below the specified. + */ + if (sysctl_anon_min_ratio) { + unsigned long reclaimable_anon; + + reclaimable_anon = + node_page_state(pgdat, NR_ACTIVE_ANON) + + node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_ISOLATED_ANON); + + sc->anon_below_min = reclaimable_anon < sysctl_anon_min_ratio_kb; + } else + sc->anon_below_min = 0; + + /* + * Check the number of clean file pages to protect them from + * reclaiming if their amount is below the specified. + */ + if (sysctl_clean_low_ratio || sysctl_clean_min_ratio) { + unsigned long reclaimable_file, dirty, clean; + + reclaimable_file = + node_page_state(pgdat, NR_ACTIVE_FILE) + + node_page_state(pgdat, NR_INACTIVE_FILE) + + node_page_state(pgdat, NR_ISOLATED_FILE); + dirty = node_page_state(pgdat, NR_FILE_DIRTY); + /* + * node_page_state() sum can go out of sync since + * all the values are not read at once. + */ + if (likely(reclaimable_file > dirty)) + clean = reclaimable_file - dirty; + else + clean = 0; + + sc->clean_below_low = clean < sysctl_clean_low_ratio_kb; + sc->clean_below_min = clean < sysctl_clean_min_ratio_kb; + } else { + sc->clean_below_low = 0; + sc->clean_below_min = 0; + } +} + #ifdef CONFIG_LRU_GEN #ifdef CONFIG_LRU_GEN_ENABLED @@ -4664,11 +4797,21 @@ static int get_tier_idx(struct lruvec *lruvec, int type) return tier - 1; } -static int get_type_to_scan(struct lruvec *lruvec, int swappiness) +static int get_type_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness) { struct ctrl_pos sp, pv; - if (swappiness <= MIN_SWAPPINESS + 1) + if (swappiness == MIN_SWAPPINESS) + return LRU_GEN_FILE; + + if (sc->clean_below_min) + return LRU_GEN_ANON; + if (sc->anon_below_min) + return LRU_GEN_FILE; + if (sc->clean_below_low) + return LRU_GEN_ANON; + + if (swappiness == MIN_SWAPPINESS + 1) return LRU_GEN_FILE; if (swappiness >= MAX_SWAPPINESS) @@ -4687,7 +4830,7 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw int *type_scanned, struct list_head *list) { int i; - int type = get_type_to_scan(lruvec, swappiness); + int type = get_type_to_scan(lruvec, sc, swappiness); for_each_evictable_type(i, swappiness) { int scanned; @@ -4930,6 +5073,12 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); + prepare_workingset_protection(pgdat, sc); + + if (sysctl_workingset_protection && sc->clean_below_min && + !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) + return 0; + /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ if (mem_cgroup_below_min(NULL, memcg)) return MEMCG_LRU_YOUNG; @@ -6079,6 +6228,8 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) prepare_scan_control(pgdat, sc); + prepare_workingset_protection(pgdat, sc); + shrink_node_memcgs(pgdat, sc); flush_reclaim_state(sc); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 46750c96d08e..cf08811a2a6b 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -632,7 +632,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) * having to remove and re-insert us on the wait queue. */ for (;;) { - prepare_to_wait_exclusive(sk_sleep(sk), &wait, + prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); release_sock(sk); if (reqsk_queue_empty(&icsk->icsk_accept_queue)) diff --git a/scripts/Makefile.thinlto b/scripts/Makefile.thinlto new file mode 100644 index 000000000000..ec98fa2ead3b --- /dev/null +++ b/scripts/Makefile.thinlto @@ -0,0 +1,38 @@ +PHONY := __default +__default: + +include include/config/auto.conf +include $(srctree)/scripts/Kbuild.include +include $(srctree)/scripts/Makefile.lib + +native-objs := $(patsubst %.o,%.thinlto-native.o,$(call read-file, vmlinux.thinlto-index)) + +__default: $(native-objs) + +# Generate .thinlto-native.o (obj) from .o (bitcode) and .thinlto.bc (summary) files +# --------------------------------------------------------------------------- +quiet_cmd_cc_o_bc = CC $(quiet_modtag) $@ + cmd_cc_o_bc = \ + $(CC) $(_c_flags) -fno-lto -Wno-unused-command-line-argument \ + -fthinlto-index=$(word 2, $^) -c -o $@ $< + +targets += $(native-objs) +$(native-objs): %.thinlto-native.o: %.o %.o.thinlto.bc FORCE + $(call if_changed,cc_o_bc) + +# Add FORCE to the prerequisites of a target to force it to be always rebuilt. +# --------------------------------------------------------------------------- + +PHONY += FORCE +FORCE: + +# Read all saved command lines and dependencies for the $(targets) we +# may be building above, using $(if_changed{,_dep}). As an +# optimization, we don't need to read them if the target does not +# exist, we will rebuild anyway in that case. + +existing-targets := $(wildcard $(sort $(targets))) + +-include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd) + +.PHONY: $(PHONY) diff --git a/scripts/Makefile.vmlinux_a b/scripts/Makefile.vmlinux_a new file mode 100644 index 000000000000..73c9545de7cf --- /dev/null +++ b/scripts/Makefile.vmlinux_a @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0-only + +PHONY := __default +__default: vmlinux.a + +include include/config/auto.conf +include $(srctree)/scripts/Kbuild.include +include $(srctree)/scripts/Makefile.lib + +# Link of built-in-fixup.a +# --------------------------------------------------------------------------- + +# '$(AR) mPi' needs 'T' to workaround the bug of llvm-ar <= 14 +quiet_cmd_ar_builtin_fixup = AR $@ + cmd_ar_builtin_fixup = \ + rm -f $@; \ + $(AR) cDPrST $@ $(KBUILD_VMLINUX_OBJS); \ + $(AR) mPiT $$($(AR) t $@ | sed -n 1p) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) + +targets += built-in-fixup.a +built-in-fixup.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt FORCE + $(call if_changed,ar_builtin_fixup) + +ifdef CONFIG_LTO_CLANG_THIN_DIST + +quiet_cmd_builtin.order = GEN $@ + cmd_builtin.order = $(AR) t $< > $@ + +targets += builtin.order +builtin.order: built-in-fixup.a FORCE + $(call if_changed,builtin.order) + +quiet_cmd_ld_thinlto_index = LD $@ + cmd_ld_thinlto_index = \ + $(LD) $(KBUILD_LDFLAGS) -r --thinlto-index-only=$@ @$< + +targets += vmlinux.thinlto-index +vmlinux.thinlto-index: builtin.order FORCE + $(call if_changed,ld_thinlto_index) + +quiet_cmd_ar_vmlinux.a = GEN $@ + cmd_ar_vmlinux.a = \ + rm -f $@; \ + while read -r obj; do \ + if grep -q $${obj} $(word 2, $^); then \ + echo $${obj%.o}.thinlto-native.o; \ + else \ + echo $${obj}; \ + fi; \ + done < $< | xargs $(AR) cDPrS $@ + +targets += vmlinux.a +vmlinux.a: builtin.order vmlinux.thinlto-index FORCE + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.thinlto + $(call if_changed,ar_vmlinux.a) + +else + +# vmlinux.a +# --------------------------------------------------------------------------- + +targets += vmlinux.a +vmlinux.a: built-in-fixup.a FORCE + $(call if_changed,copy) + +endif + +# Add FORCE to the prerequisites of a target to force it to be always rebuilt. +# --------------------------------------------------------------------------- + +PHONY += FORCE +FORCE: + +# Read all saved command lines and dependencies for the $(targets) we +# may be building above, using $(if_changed{,_dep}). As an +# optimization, we don't need to read them if the target does not +# exist, we will rebuild anyway in that case. + +existing-targets := $(wildcard $(sort $(targets))) + +-include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd) + +.PHONY: $(PHONY) diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 5ca7c268294e..8b01746c9ce6 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1473,13 +1473,22 @@ static void extract_crcs_for_object(const char *object, struct module *mod) char cmd_file[PATH_MAX]; char *buf, *p; const char *base; - int dirlen, ret; + int dirlen, baselen_without_suffix, ret; base = get_basename(object); dirlen = base - object; - ret = snprintf(cmd_file, sizeof(cmd_file), "%.*s.%s.cmd", - dirlen, object, base); + baselen_without_suffix = strlen(object) - dirlen - strlen(".o"); + + /* + * When CONFIG_LTO_CLANG_THIN_DIST=y, the ELF is *.thinlto-native.o + * but the symbol CRCs are recorded in *.o.cmd file. + */ + if (strends(object, ".thinlto-native.o")) + baselen_without_suffix -= strlen(".thinlto-native"); + + ret = snprintf(cmd_file, sizeof(cmd_file), "%.*s.%.*s.o.cmd", + dirlen, object, baselen_without_suffix, base); if (ret >= sizeof(cmd_file)) { error("%s: too long path was truncated\n", cmd_file); return; -- 2.51.0 From 683e2cc179446131f136533d812082dde1b8a2bc Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 19:04:03 +0200 Subject: [PATCH] adios: Fixup for block changes in 6.16.1 Signed-off-by: Peter Jung --- block/adios.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/block/adios.c b/block/adios.c index bcc90564b9ce..98c229dad177 100644 --- a/block/adios.c +++ b/block/adios.c @@ -1327,24 +1327,19 @@ static int adios_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { } // Initialize the scheduler-specific data when initializing the request queue -static int adios_init_sched(struct request_queue *q, struct elevator_type *e) { +static int adios_init_sched(struct request_queue *q, struct elevator_queue *eq) { struct adios_data *ad; - struct elevator_queue *eq; int ret = -ENOMEM; u8 optype = 0; - eq = elevator_alloc(q, e); - if (!eq) { - pr_err("adios: Failed to allocate the elevator\n"); - return ret; - } - ad = kzalloc_node(sizeof(*ad), GFP_KERNEL, q->node); if (!ad) { pr_err("adios: Failed to create adios_data\n"); goto put_eq; } + eq->elevator_data = ad; + // Create a memory pool for adios_rq_data ad->rq_data_pool = kmem_cache_create("rq_data_pool", sizeof(struct adios_rq_data), -- 2.51.0 From 5f25f97032586b1741f9603db23c479e6dc544c4 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 18:20:28 +0200 Subject: [PATCH 5/6] fixes Signed-off-by: Peter Jung --- drivers/gpu/drm/drm_atomic_uapi.c | 51 ++++++++++++++++-------- drivers/gpu/drm/i915/display/intel_dsb.c | 4 ++ drivers/watchdog/intel_oc_wdt.c | 8 ++-- fs/proc/generic.c | 36 +++++++++-------- net/ipv4/route.c | 7 +++- scripts/package/PKGBUILD | 5 +++ 6 files changed, 73 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index c2726af6698e..2ae41a522e92 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1068,7 +1068,6 @@ int drm_atomic_set_property(struct drm_atomic_state *state, struct drm_plane *plane = obj_to_plane(obj); struct drm_plane_state *plane_state; struct drm_mode_config *config = &plane->dev->mode_config; - const struct drm_plane_helper_funcs *plane_funcs = plane->helper_private; plane_state = drm_atomic_get_plane_state(state, plane); if (IS_ERR(plane_state)) { @@ -1084,21 +1083,8 @@ int drm_atomic_set_property(struct drm_atomic_state *state, ret = drm_atomic_plane_get_property(plane, plane_state, prop, &old_val); ret = drm_atomic_check_prop_changes(ret, old_val, prop_value, prop); - } - - /* ask the driver if this non-primary plane is supported */ - if (plane->type != DRM_PLANE_TYPE_PRIMARY) { - ret = -EINVAL; - - if (plane_funcs && plane_funcs->atomic_async_check) - ret = plane_funcs->atomic_async_check(plane, state, true); - - if (ret) { - drm_dbg_atomic(prop->dev, - "[PLANE:%d:%s] does not support async flips\n", - obj->id, plane->name); - break; - } + if (ret) + break; } } @@ -1394,6 +1380,10 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, int ret = 0; unsigned int i, j, num_fences; bool async_flip = false; + struct drm_plane *plane; + struct drm_plane_state *old_plane_state = NULL; + struct drm_plane_state *new_plane_state = NULL; + u64 fb_id = 0; /* disallow for drivers not supporting atomic: */ if (!drm_core_check_feature(dev, DRIVER_ATOMIC)) @@ -1521,6 +1511,35 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, copied_props++; } + if (async_flip && obj->type == DRM_MODE_OBJECT_PLANE && + obj_to_plane(obj)->type != DRM_PLANE_TYPE_PRIMARY) { + /* need to ask the driver if this plane is supported */ + plane = obj_to_plane(obj); + old_plane_state = drm_atomic_get_old_plane_state(state, plane); + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + ret = drm_atomic_plane_get_property(plane, new_plane_state, + dev->mode_config.prop_fb_id, + &fb_id); + if (ret) + break; + /* + * Only do the check if the plane was or is enabled. + * Note that the new state doesn't have "visible" set yet, + * so this uses fb_id instead. + */ + if (old_plane_state->visible || fb_id) + ret = -EINVAL; + if (ret && plane->helper_private && + plane->helper_private->atomic_async_check) { + ret = plane->helper_private->atomic_async_check(plane, state, true); + } + if (ret) { + drm_dbg_atomic(dev, "[PLANE:%d:%s] does not support async flips\n", + obj->id, plane->name); + break; + } + } + drm_mode_object_put(obj); } diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c index 481488d1fe67..271229500c62 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.c +++ b/drivers/gpu/drm/i915/display/intel_dsb.c @@ -808,6 +808,10 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state, if (!display->params.enable_dsb) return NULL; + /* TODO: DSB is broken in Xe KMD, so disabling it until fixed */ + if (!IS_ENABLED(I915)) + return NULL; + dsb = kzalloc(sizeof(*dsb), GFP_KERNEL); if (!dsb) goto out; diff --git a/drivers/watchdog/intel_oc_wdt.c b/drivers/watchdog/intel_oc_wdt.c index 7c0551106981..a39892c10770 100644 --- a/drivers/watchdog/intel_oc_wdt.c +++ b/drivers/watchdog/intel_oc_wdt.c @@ -41,6 +41,7 @@ struct intel_oc_wdt { struct watchdog_device wdd; struct resource *ctrl_res; + struct watchdog_info info; bool locked; }; @@ -115,7 +116,6 @@ static const struct watchdog_ops intel_oc_wdt_ops = { static int intel_oc_wdt_setup(struct intel_oc_wdt *oc_wdt) { - struct watchdog_info *info; unsigned long val; val = inl(INTEL_OC_WDT_CTRL_REG(oc_wdt)); @@ -134,7 +134,6 @@ static int intel_oc_wdt_setup(struct intel_oc_wdt *oc_wdt) set_bit(WDOG_HW_RUNNING, &oc_wdt->wdd.status); if (oc_wdt->locked) { - info = (struct watchdog_info *)&intel_oc_wdt_info; /* * Set nowayout unconditionally as we cannot stop * the watchdog. @@ -145,7 +144,7 @@ static int intel_oc_wdt_setup(struct intel_oc_wdt *oc_wdt) * and inform the core we can't change it. */ oc_wdt->wdd.timeout = (val & INTEL_OC_WDT_TOV) + 1; - info->options &= ~WDIOF_SETTIMEOUT; + oc_wdt->info.options &= ~WDIOF_SETTIMEOUT; dev_info(oc_wdt->wdd.parent, "Register access locked, heartbeat fixed at: %u s\n", @@ -193,7 +192,8 @@ static int intel_oc_wdt_probe(struct platform_device *pdev) wdd->min_timeout = INTEL_OC_WDT_MIN_TOV; wdd->max_timeout = INTEL_OC_WDT_MAX_TOV; wdd->timeout = INTEL_OC_WDT_DEF_TOV; - wdd->info = &intel_oc_wdt_info; + oc_wdt->info = intel_oc_wdt_info; + wdd->info = &oc_wdt->info; wdd->ops = &intel_oc_wdt_ops; wdd->parent = dev; diff --git a/fs/proc/generic.c b/fs/proc/generic.c index e0e50914ab25..8bc2b960614c 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -364,6 +364,23 @@ static const struct inode_operations proc_dir_inode_operations = { .setattr = proc_notify_change, }; +static void pde_set_flags(struct proc_dir_entry *pde) +{ + if (!pde->proc_ops) + return; + + if (pde->proc_ops->proc_flags & PROC_ENTRY_PERMANENT) + pde->flags |= PROC_ENTRY_PERMANENT; + if (pde->proc_ops->proc_read_iter) + pde->flags |= PROC_ENTRY_proc_read_iter; +#ifdef CONFIG_COMPAT + if (pde->proc_ops->proc_compat_ioctl) + pde->flags |= PROC_ENTRY_proc_compat_ioctl; +#endif + if (pde->proc_ops->proc_lseek) + pde->flags |= PROC_ENTRY_proc_lseek; +} + /* returns the registered entry, or frees dp and returns NULL on failure */ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, struct proc_dir_entry *dp) @@ -371,6 +388,8 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, if (proc_alloc_inum(&dp->low_ino)) goto out_free_entry; + pde_set_flags(dp); + write_lock(&proc_subdir_lock); dp->parent = dir; if (pde_subdir_insert(dir, dp) == false) { @@ -559,20 +578,6 @@ struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode, return p; } -static void pde_set_flags(struct proc_dir_entry *pde) -{ - if (pde->proc_ops->proc_flags & PROC_ENTRY_PERMANENT) - pde->flags |= PROC_ENTRY_PERMANENT; - if (pde->proc_ops->proc_read_iter) - pde->flags |= PROC_ENTRY_proc_read_iter; -#ifdef CONFIG_COMPAT - if (pde->proc_ops->proc_compat_ioctl) - pde->flags |= PROC_ENTRY_proc_compat_ioctl; -#endif - if (pde->proc_ops->proc_lseek) - pde->flags |= PROC_ENTRY_proc_lseek; -} - struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *parent, const struct proc_ops *proc_ops, void *data) @@ -583,7 +588,6 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, if (!p) return NULL; p->proc_ops = proc_ops; - pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_data); @@ -634,7 +638,6 @@ struct proc_dir_entry *proc_create_seq_private(const char *name, umode_t mode, p->proc_ops = &proc_seq_ops; p->seq_ops = ops; p->state_size = state_size; - pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_seq_private); @@ -665,7 +668,6 @@ struct proc_dir_entry *proc_create_single_data(const char *name, umode_t mode, return NULL; p->proc_ops = &proc_single_ops; p->single_show = show; - pde_set_flags(p); return proc_register(parent, p); } EXPORT_SYMBOL(proc_create_single_data); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 7b8e80c4f1d9..25ee7907d212 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -2586,6 +2586,7 @@ static struct rtable *__mkroute_output(const struct fib_result *res, do_cache = true; if (type == RTN_BROADCAST) { flags |= RTCF_BROADCAST | RTCF_LOCAL; + fi = NULL; } else if (type == RTN_MULTICAST) { flags |= RTCF_MULTICAST | RTCF_LOCAL; if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr, @@ -2655,8 +2656,12 @@ static struct rtable *__mkroute_output(const struct fib_result *res, rth->dst.output = ip_mc_output; RT_CACHE_STAT_INC(out_slow_mc); } + if (type == RTN_BROADCAST && res->fi) { + /* ensure MTU value for broadcast routes is retained */ + ip_dst_init_metrics(&rth->dst, res->fi->fib_metrics); + } #ifdef CONFIG_IP_MROUTE - if (type == RTN_MULTICAST) { + else if (type == RTN_MULTICAST) { if (IN_DEV_MFORWARD(in_dev) && !ipv4_is_local_multicast(fl4->daddr)) { rth->dst.input = ip_mr_input; diff --git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD index 452374d63c24..08f80d7c5df0 100644 --- a/scripts/package/PKGBUILD +++ b/scripts/package/PKGBUILD @@ -90,6 +90,11 @@ _package-headers() { "${srctree}/scripts/package/install-extmod-build" "${builddir}" fi + # required when DEBUG_INFO_BTF_MODULES is enabled + if [ -f tools/bpf/resolve_btfids/resolve_btfids ]; then + install -Dt "$builddir/tools/bpf/resolve_btfids" tools/bpf/resolve_btfids/resolve_btfids + fi + echo "Installing System.map and config..." mkdir -p "${builddir}" cp System.map "${builddir}/System.map" -- 2.51.0 From efbe60263ec2eaa9bd75ce9d5f8eeeac7b2aa85d Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 29 Aug 2025 20:24:51 +0200 Subject: [PATCH] Revert "drm: don't run atomic_async_check for disabled planes" This reverts commit f0c95743e147bbb98e8c81bd38e3bada8ae2d0f6. --- drivers/gpu/drm/drm_atomic_uapi.c | 51 ++++++++++--------------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index e8b6c87bdd45..ecc73d52bfae 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1069,6 +1069,7 @@ int drm_atomic_set_property(struct drm_atomic_state *state, struct drm_plane *plane = obj_to_plane(obj); struct drm_plane_state *plane_state; struct drm_mode_config *config = &plane->dev->mode_config; + const struct drm_plane_helper_funcs *plane_funcs = plane->helper_private; plane_state = drm_atomic_get_plane_state(state, plane); if (IS_ERR(plane_state)) { @@ -1084,8 +1085,21 @@ int drm_atomic_set_property(struct drm_atomic_state *state, ret = drm_atomic_plane_get_property(plane, plane_state, prop, &old_val); ret = drm_atomic_check_prop_changes(ret, old_val, prop_value, prop); - if (ret) - break; + } + + /* ask the driver if this non-primary plane is supported */ + if (plane->type != DRM_PLANE_TYPE_PRIMARY) { + ret = -EINVAL; + + if (plane_funcs && plane_funcs->atomic_async_check) + ret = plane_funcs->atomic_async_check(plane, state, true); + + if (ret) { + drm_dbg_atomic(prop->dev, + "[PLANE:%d:%s] does not support async flips\n", + obj->id, plane->name); + break; + } } } @@ -1381,10 +1395,6 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, int ret = 0; unsigned int i, j, num_fences; bool async_flip = false; - struct drm_plane *plane; - struct drm_plane_state *old_plane_state = NULL; - struct drm_plane_state *new_plane_state = NULL; - u64 fb_id = 0; /* disallow for drivers not supporting atomic: */ if (!drm_core_check_feature(dev, DRIVER_ATOMIC)) @@ -1512,35 +1522,6 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, copied_props++; } - if (async_flip && obj->type == DRM_MODE_OBJECT_PLANE && - obj_to_plane(obj)->type != DRM_PLANE_TYPE_PRIMARY) { - /* need to ask the driver if this plane is supported */ - plane = obj_to_plane(obj); - old_plane_state = drm_atomic_get_old_plane_state(state, plane); - new_plane_state = drm_atomic_get_new_plane_state(state, plane); - ret = drm_atomic_plane_get_property(plane, new_plane_state, - dev->mode_config.prop_fb_id, - &fb_id); - if (ret) - break; - /* - * Only do the check if the plane was or is enabled. - * Note that the new state doesn't have "visible" set yet, - * so this uses fb_id instead. - */ - if (old_plane_state->visible || fb_id) - ret = -EINVAL; - if (ret && plane->helper_private && - plane->helper_private->atomic_async_check) { - ret = plane->helper_private->atomic_async_check(plane, state, true); - } - if (ret) { - drm_dbg_atomic(dev, "[PLANE:%d:%s] does not support async flips\n", - obj->id, plane->name); - break; - } - } - drm_mode_object_put(obj); } -- 2.51.0 From c910f69028975c111de6abe34f08fa1b41205711 Mon Sep 17 00:00:00 2001 From: Xaver Hugl Date: Fri, 22 Aug 2025 17:28:49 +0200 Subject: [PATCH] drm: re-allow no-op changes on non-primary planes in async flips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit fd40a63c63a1 ("drm/atomic: Let drivers decide which planes to async flip") unintentionally disallowed no-op changes on non-primary planes that the driver doesn't allow async flips on. This broke async flips for compositors that disable the cursor plane in every async atomic commit. To fix that, change drm_atomic_set_property to again only run atomic_async_check if the plane would actually be changed by the atomic commit. Fixes: fd40a63c63a1 ("drm/atomic: Let drivers decide which planes to async flip") Closes: https://gitlab.freedesktop.org/drm/amd/-/issues/4263 Signed-off-by: Xaver Hugl Reviewed-by: André Almeida Link: https://lore.kernel.org/r/20250822152849.87843-1-xaver.hugl@kde.org [andrealmeid: fix checkpatch warning] Signed-off-by: André Almeida --- drivers/gpu/drm/drm_atomic_uapi.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index ecc73d52bfae..85dbdaa4a2e2 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1078,19 +1078,20 @@ int drm_atomic_set_property(struct drm_atomic_state *state, } if (async_flip) { - /* check if the prop does a nop change */ - if ((prop != config->prop_fb_id && - prop != config->prop_in_fence_fd && - prop != config->prop_fb_damage_clips)) { - ret = drm_atomic_plane_get_property(plane, plane_state, - prop, &old_val); - ret = drm_atomic_check_prop_changes(ret, old_val, prop_value, prop); - } + /* no-op changes are always allowed */ + ret = drm_atomic_plane_get_property(plane, plane_state, + prop, &old_val); + ret = drm_atomic_check_prop_changes(ret, old_val, prop_value, prop); - /* ask the driver if this non-primary plane is supported */ - if (plane->type != DRM_PLANE_TYPE_PRIMARY) { - ret = -EINVAL; + /* fail everything that isn't no-op or a pure flip */ + if (ret && prop != config->prop_fb_id && + prop != config->prop_in_fence_fd && + prop != config->prop_fb_damage_clips) { + break; + } + if (ret && plane->type != DRM_PLANE_TYPE_PRIMARY) { + /* ask the driver if this non-primary plane is supported */ if (plane_funcs && plane_funcs->atomic_async_check) ret = plane_funcs->atomic_async_check(plane, state, true); -- 2.51.0 From 48d4a9f054ec6d82192b2dccadd5b450f34d1cad Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Thu, 28 Aug 2025 18:20:38 +0200 Subject: [PATCH 6/6] t2 Signed-off-by: Peter Jung --- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 + drivers/gpu/drm/i915/display/intel_ddi.c | 3 + drivers/gpu/drm/i915/display/intel_fbdev.c | 6 +- drivers/gpu/drm/i915/display/intel_quirks.c | 15 + drivers/gpu/drm/i915/display/intel_quirks.h | 1 + drivers/gpu/vga/vga_switcheroo.c | 7 +- drivers/hid/Kconfig | 11 +- drivers/hid/Makefile | 4 + drivers/hid/dockchannel-hid/Kconfig | 14 + drivers/hid/dockchannel-hid/Makefile | 6 + drivers/hid/dockchannel-hid/dockchannel-hid.c | 1213 +++++++++++++++++ drivers/hid/hid-apple.c | 62 +- drivers/hid/hid-core.c | 11 +- drivers/hid/hid-ids.h | 25 +- drivers/hid/hid-magicmouse.c | 924 +++++++++++-- drivers/hid/hid-multitouch.c | 62 +- drivers/hid/hid-quirks.c | 9 +- drivers/hid/spi-hid/Kconfig | 26 + drivers/hid/spi-hid/Makefile | 10 + drivers/hid/spi-hid/spi-hid-apple-core.c | 1194 ++++++++++++++++ drivers/hid/spi-hid/spi-hid-apple-of.c | 151 ++ drivers/hid/spi-hid/spi-hid-apple.h | 35 + drivers/hwmon/applesmc.c | 1138 ++++++++++++---- drivers/pci/vgaarb.c | 1 + drivers/platform/x86/apple-gmux.c | 18 + drivers/soc/apple/Kconfig | 24 + drivers/soc/apple/Makefile | 6 + drivers/soc/apple/dockchannel.c | 406 ++++++ drivers/soc/apple/rtkit-helper.c | 151 ++ drivers/staging/Kconfig | 2 + drivers/staging/Makefile | 1 + drivers/staging/apple-bce/Kconfig | 18 + drivers/staging/apple-bce/Makefile | 28 + drivers/staging/apple-bce/apple_bce.c | 445 ++++++ drivers/staging/apple-bce/apple_bce.h | 41 + drivers/staging/apple-bce/audio/audio.c | 711 ++++++++++ drivers/staging/apple-bce/audio/audio.h | 125 ++ drivers/staging/apple-bce/audio/description.h | 42 + drivers/staging/apple-bce/audio/pcm.c | 308 +++++ drivers/staging/apple-bce/audio/pcm.h | 16 + drivers/staging/apple-bce/audio/protocol.c | 347 +++++ drivers/staging/apple-bce/audio/protocol.h | 147 ++ .../staging/apple-bce/audio/protocol_bce.c | 226 +++ .../staging/apple-bce/audio/protocol_bce.h | 72 + drivers/staging/apple-bce/mailbox.c | 155 +++ drivers/staging/apple-bce/mailbox.h | 53 + drivers/staging/apple-bce/queue.c | 390 ++++++ drivers/staging/apple-bce/queue.h | 177 +++ drivers/staging/apple-bce/queue_dma.c | 220 +++ drivers/staging/apple-bce/queue_dma.h | 50 + drivers/staging/apple-bce/vhci/command.h | 204 +++ drivers/staging/apple-bce/vhci/queue.c | 268 ++++ drivers/staging/apple-bce/vhci/queue.h | 76 ++ drivers/staging/apple-bce/vhci/transfer.c | 673 +++++++++ drivers/staging/apple-bce/vhci/transfer.h | 75 + drivers/staging/apple-bce/vhci/vhci.c | 763 +++++++++++ drivers/staging/apple-bce/vhci/vhci.h | 52 + include/linux/hid.h | 6 +- include/linux/soc/apple/dockchannel.h | 26 + 59 files changed, 10795 insertions(+), 458 deletions(-) create mode 100644 drivers/hid/dockchannel-hid/Kconfig create mode 100644 drivers/hid/dockchannel-hid/Makefile create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid.c create mode 100644 drivers/hid/spi-hid/Kconfig create mode 100644 drivers/hid/spi-hid/Makefile create mode 100644 drivers/hid/spi-hid/spi-hid-apple-core.c create mode 100644 drivers/hid/spi-hid/spi-hid-apple-of.c create mode 100644 drivers/hid/spi-hid/spi-hid-apple.h create mode 100644 drivers/soc/apple/dockchannel.c create mode 100644 drivers/soc/apple/rtkit-helper.c create mode 100644 drivers/staging/apple-bce/Kconfig create mode 100644 drivers/staging/apple-bce/Makefile create mode 100644 drivers/staging/apple-bce/apple_bce.c create mode 100644 drivers/staging/apple-bce/apple_bce.h create mode 100644 drivers/staging/apple-bce/audio/audio.c create mode 100644 drivers/staging/apple-bce/audio/audio.h create mode 100644 drivers/staging/apple-bce/audio/description.h create mode 100644 drivers/staging/apple-bce/audio/pcm.c create mode 100644 drivers/staging/apple-bce/audio/pcm.h create mode 100644 drivers/staging/apple-bce/audio/protocol.c create mode 100644 drivers/staging/apple-bce/audio/protocol.h create mode 100644 drivers/staging/apple-bce/audio/protocol_bce.c create mode 100644 drivers/staging/apple-bce/audio/protocol_bce.h create mode 100644 drivers/staging/apple-bce/mailbox.c create mode 100644 drivers/staging/apple-bce/mailbox.h create mode 100644 drivers/staging/apple-bce/queue.c create mode 100644 drivers/staging/apple-bce/queue.h create mode 100644 drivers/staging/apple-bce/queue_dma.c create mode 100644 drivers/staging/apple-bce/queue_dma.h create mode 100644 drivers/staging/apple-bce/vhci/command.h create mode 100644 drivers/staging/apple-bce/vhci/queue.c create mode 100644 drivers/staging/apple-bce/vhci/queue.h create mode 100644 drivers/staging/apple-bce/vhci/transfer.c create mode 100644 drivers/staging/apple-bce/vhci/transfer.h create mode 100644 drivers/staging/apple-bce/vhci/vhci.c create mode 100644 drivers/staging/apple-bce/vhci/vhci.h create mode 100644 include/linux/soc/apple/dockchannel.h diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index a5db963da610..90d83b721ed3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2315,6 +2315,9 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, int ret, retry = 0, i; bool supports_atomic = false; + if (vga_switcheroo_client_probe_defer(pdev)) + return -EPROBE_DEFER; + if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA || (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) { if (drm_firmware_drivers_only() && amdgpu_modeset == -1) diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index d58f8fc37326..38df2d26baf2 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -4832,6 +4832,9 @@ static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dig_port) if (dig_port->ddi_a_4_lanes) return false; + if (intel_has_quirk(display, QUIRK_DDI_A_FORCE_4_LANES)) + return true; + /* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only * supported configuration */ diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index 2dc4029d71ed..eaa82801ef01 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -224,10 +224,10 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, ifbdev->fb = NULL; if (fb && - (sizes->fb_width > fb->base.width || - sizes->fb_height > fb->base.height)) { + (sizes->fb_width != fb->base.width || + sizes->fb_height != fb->base.height)) { drm_dbg_kms(display->drm, - "BIOS fb too small (%dx%d), we require (%dx%d)," + "BIOS fb not valid (%dx%d), we require (%dx%d)," " releasing it\n", fb->base.width, fb->base.height, sizes->fb_width, sizes->fb_height); diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index a32fae510ed2..662465733ddb 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -66,6 +66,18 @@ static void quirk_increase_ddi_disabled_time(struct intel_display *display) drm_info(display->drm, "Applying Increase DDI Disabled quirk\n"); } +/* + * In some cases, the firmware might not set the lane count to 4 (for example, + * when booting in some dual GPU Macs with the dGPU as the default GPU), this + * quirk is used to force it as otherwise it might not be possible to compute a + * valid link configuration. + */ +static void quirk_ddi_a_force_4_lanes(struct intel_display *display) +{ + intel_set_quirk(display, QUIRK_DDI_A_FORCE_4_LANES); + drm_info(display->drm, "Applying DDI A Forced 4 Lanes quirk\n"); +} + static void quirk_no_pps_backlight_power_hook(struct intel_display *display) { intel_set_quirk(display, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK); @@ -231,6 +243,9 @@ static struct intel_quirk intel_quirks[] = { { 0x3184, 0x1019, 0xa94d, quirk_increase_ddi_disabled_time }, /* HP Notebook - 14-r206nv */ { 0x0f31, 0x103c, 0x220f, quirk_invert_brightness }, + + /* Apple MacBookPro15,1 */ + { 0x3e9b, 0x106b, 0x0176, quirk_ddi_a_force_4_lanes }, }; static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h index cafdebda7535..a5296f82776e 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.h +++ b/drivers/gpu/drm/i915/display/intel_quirks.h @@ -20,6 +20,7 @@ enum intel_quirk_id { QUIRK_LVDS_SSC_DISABLE, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK, QUIRK_FW_SYNC_LEN, + QUIRK_DDI_A_FORCE_4_LANES, }; void intel_init_quirks(struct intel_display *display); diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c index 18f2c92beff8..3de1bca45ed2 100644 --- a/drivers/gpu/vga/vga_switcheroo.c +++ b/drivers/gpu/vga/vga_switcheroo.c @@ -438,12 +438,7 @@ find_active_client(struct list_head *head) bool vga_switcheroo_client_probe_defer(struct pci_dev *pdev) { if ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY) { - /* - * apple-gmux is needed on pre-retina MacBook Pro - * to probe the panel if pdev is the inactive GPU. - */ - if (apple_gmux_present() && pdev != vga_default_device() && - !vgasr_priv.handler_flags) + if (apple_gmux_present() && !vgasr_priv.handler_flags) return true; } diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 5bc1f086b855..1c2522d95934 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -129,7 +129,7 @@ config HID_APPLE tristate "Apple {i,Power,Mac}Books" depends on LEDS_CLASS depends on NEW_LEDS - default !EXPERT + default !EXPERT || SPI_HID_APPLE help Support for some Apple devices which less or more break HID specification. @@ -727,11 +727,13 @@ config LOGIWHEELS_FF config HID_MAGICMOUSE tristate "Apple Magic Mouse/Trackpad multi-touch support" + default SPI_HID_APPLE help Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" @@ -781,6 +783,7 @@ config HID_MULTITOUCH Say Y here if you have one of the following devices: - 3M PCT touch screens - ActionStar dual touch panels + - Apple Touch Bar on x86 MacBook Pros - Atmel panels - Cando dual touch panels - Chunghwa panels @@ -1445,4 +1448,8 @@ endif # HID source "drivers/hid/usbhid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 958f67193c85..f67db3d5d18f 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -176,3 +176,7 @@ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig new file mode 100644 index 000000000000..8a81d551a83d --- /dev/null +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + default ARCH_APPLE + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile new file mode 100644 index 000000000000..7dba766b047f --- /dev/null +++ b/drivers/hid/dockchannel-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c new file mode 100644 index 000000000000..a712a724ded3 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -0,0 +1,1213 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, HID_FEATURE_REPORT, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%d > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case HID_INPUT_REPORT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%d > %ld)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static void dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 3bed3f0c90c2..21ff5d1da011 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -486,6 +486,7 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, table = apple2021_fn_keys; else if (hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 || hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 || + hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT || hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213) table = macbookpro_no_esc_fn_keys; else if (hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K || @@ -498,6 +499,16 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else if (hid->product >= USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI && hid->product <= USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) table = macbookair_fn_keys; + else if (hid->bus == BUS_HOST || hid->bus == BUS_SPI) + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = macbookpro_dedicated_esc_fn_keys; + break; + default: + table = apple2021_fn_keys; + break; + } else if (hid->product < 0x21d || hid->product >= 0x300) table = powerbook_fn_keys; else @@ -911,6 +922,13 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + + if (quirks & APPLE_IGNORE_MOUSE && hdev->type == HID_TYPE_USBMOUSE) + return -ENODEV; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -934,12 +952,10 @@ static int apple_probe(struct hid_device *hdev, return ret; } - if (quirks & APPLE_RDESC_BATTERY) { - timer_setup(&asc->battery_timer, apple_battery_timer_tick, 0); - mod_timer(&asc->battery_timer, - jiffies + msecs_to_jiffies(APPLE_BATTERY_TIMEOUT_MS)); - apple_fetch_battery(hdev); - } + timer_setup(&asc->battery_timer, apple_battery_timer_tick, 0); + mod_timer(&asc->battery_timer, + jiffies + msecs_to_jiffies(APPLE_BATTERY_TIMEOUT_MS)); + apple_fetch_battery(hdev); if (quirks & APPLE_BACKLIGHT_CTL) apple_backlight_init(hdev); @@ -953,9 +969,7 @@ static int apple_probe(struct hid_device *hdev, return 0; out_err: - if (quirks & APPLE_RDESC_BATTERY) - timer_delete_sync(&asc->battery_timer); - + timer_delete_sync(&asc->battery_timer); hid_hw_stop(hdev); return ret; } @@ -964,8 +978,7 @@ static void apple_remove(struct hid_device *hdev) { struct apple_sc *asc = hid_get_drvdata(hdev); - if (asc->quirks & APPLE_RDESC_BATTERY) - timer_delete_sync(&asc->battery_timer); + timer_delete_sync(&asc->battery_timer); hid_hw_stop(hdev); } @@ -1133,21 +1146,28 @@ static const struct hid_device_id apple_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS), .driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT), + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ANSI), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ISO), @@ -1175,6 +1195,10 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT), .driver_data = APPLE_MAGIC_BACKLIGHT }, diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 4fd3fff4661c..cd6c1cc7d010 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -468,7 +468,10 @@ static int hid_parser_global(struct hid_parser *parser, struct hid_item *item) case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; @@ -2307,6 +2310,12 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_I2C: bus = "I2C"; break; + case BUS_SPI: + bus = "SPI"; + break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 9458af1c2630..bc2fba4e7a4d 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -93,6 +93,8 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -173,14 +175,15 @@ #define USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI 0x0272 #define USB_DEVICE_ID_APPLE_WELLSPRING9_ISO 0x0273 #define USB_DEVICE_ID_APPLE_WELLSPRING9_JIS 0x0274 -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K 0x027a -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 0x027b -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 0x027c -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213 0x027d -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K 0x027e -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223 0x027f -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K 0x0280 -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F 0x0340 +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K 0x027a +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 0x027b +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 0x027c +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT 0x0278 +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213 0x027d +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K 0x027e +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223 0x027f +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K 0x0280 +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F 0x0340 #define USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY 0x030a #define USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY 0x030b #define USB_DEVICE_ID_APPLE_IRCONTROL 0x8240 @@ -194,6 +197,12 @@ #define USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021 0x029f #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASETEK 0x2433 #define USB_DEVICE_ID_ASETEK_INVICTA 0xf300 diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 226682762db3..328df8118384 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -60,8 +60,14 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 +#define MTP_REPORT_ID 0x75 +#define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_MS 60000 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -112,30 +118,156 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +#define J140K_TP_DIMENSION_X (float)12100 +#define J140K_TP_MIN_X -5318 +#define J140K_TP_MAX_X 5787 +#define J140K_TP_RES_X \ + ((J140K_TP_MAX_X - J140K_TP_MIN_X) / (J140K_TP_DIMENSION_X / 100)) +#define J140K_TP_DIMENSION_Y (float)8200 +#define J140K_TP_MIN_Y -157 +#define J140K_TP_MAX_Y 7102 +#define J140K_TP_RES_Y \ + ((J140K_TP_MAX_Y - J140K_TP_MIN_Y) / (J140K_TP_DIMENSION_Y / 100)) + +#define J132_TP_DIMENSION_X (float)13500 +#define J132_TP_MIN_X -6243 +#define J132_TP_MAX_X 6749 +#define J132_TP_RES_X \ + ((J132_TP_MAX_X - J132_TP_MIN_X) / (J132_TP_DIMENSION_X / 100)) +#define J132_TP_DIMENSION_Y (float)8400 +#define J132_TP_MIN_Y -170 +#define J132_TP_MAX_Y 7685 +#define J132_TP_RES_Y \ + ((J132_TP_MAX_Y - J132_TP_MIN_Y) / (J132_TP_DIMENSION_Y / 100)) + +#define J680_TP_DIMENSION_X (float)16000 +#define J680_TP_MIN_X -7456 +#define J680_TP_MAX_X 7976 +#define J680_TP_RES_X \ + ((J680_TP_MAX_X - J680_TP_MIN_X) / (J680_TP_DIMENSION_X / 100)) +#define J680_TP_DIMENSION_Y (float)10000 +#define J680_TP_MIN_Y -163 +#define J680_TP_MAX_Y 9283 +#define J680_TP_RES_Y \ + ((J680_TP_MAX_Y - J680_TP_MIN_Y) / (J680_TP_DIMENSION_Y / 100)) + +#define J680_ALT_TP_DIMENSION_X (float)16000 +#define J680_ALT_TP_MIN_X -7456 +#define J680_ALT_TP_MAX_X 7976 +#define J680_ALT_TP_RES_X \ + ((J680_ALT_TP_MAX_X - J680_ALT_TP_MIN_X) / (J680_ALT_TP_DIMENSION_X / 100)) +#define J680_ALT_TP_DIMENSION_Y (float)10000 +#define J680_ALT_TP_MIN_Y -163 +#define J680_ALT_TP_MAX_Y 9283 +#define J680_ALT_TP_RES_Y \ + ((J680_ALT_TP_MAX_Y - J680_ALT_TP_MIN_Y) / (J680_ALT_TP_DIMENSION_Y / 100)) + +#define J213_TP_DIMENSION_X (float)13500 +#define J213_TP_MIN_X -6243 +#define J213_TP_MAX_X 6749 +#define J213_TP_RES_X \ + ((J213_TP_MAX_X - J213_TP_MIN_X) / (J213_TP_DIMENSION_X / 100)) +#define J213_TP_DIMENSION_Y (float)8400 +#define J213_TP_MIN_Y -170 +#define J213_TP_MAX_Y 7685 +#define J213_TP_RES_Y \ + ((J213_TP_MAX_Y - J213_TP_MIN_Y) / (J213_TP_DIMENSION_Y / 100)) + +#define J214K_TP_DIMENSION_X (float)13200 +#define J214K_TP_MIN_X -6046 +#define J214K_TP_MAX_X 6536 +#define J214K_TP_RES_X \ + ((J214K_TP_MAX_X - J214K_TP_MIN_X) / (J214K_TP_DIMENSION_X / 100)) +#define J214K_TP_DIMENSION_Y (float)8200 +#define J214K_TP_MIN_Y -164 +#define J214K_TP_MAX_Y 7439 +#define J214K_TP_RES_Y \ + ((J214K_TP_MAX_Y - J214K_TP_MIN_Y) / (J214K_TP_DIMENSION_Y / 100)) + +#define J223_TP_DIMENSION_X (float)13200 +#define J223_TP_MIN_X -6046 +#define J223_TP_MAX_X 6536 +#define J223_TP_RES_X \ + ((J223_TP_MAX_X - J223_TP_MIN_X) / (J223_TP_DIMENSION_X / 100)) +#define J223_TP_DIMENSION_Y (float)8200 +#define J223_TP_MIN_Y -164 +#define J223_TP_MAX_Y 7439 +#define J223_TP_RES_Y \ + ((J223_TP_MAX_Y - J223_TP_MIN_Y) / (J223_TP_DIMENSION_Y / 100)) + +#define J230K_TP_DIMENSION_X (float)12100 +#define J230K_TP_MIN_X -5318 +#define J230K_TP_MAX_X 5787 +#define J230K_TP_RES_X \ + ((J230K_TP_MAX_X - J230K_TP_MIN_X) / (J230K_TP_DIMENSION_X / 100)) +#define J230K_TP_DIMENSION_Y (float)8200 +#define J230K_TP_MIN_Y -157 +#define J230K_TP_MAX_Y 7102 +#define J230K_TP_RES_Y \ + ((J230K_TP_MAX_Y - J230K_TP_MIN_Y) / (J230K_TP_DIMENSION_Y / 100)) + +#define J152F_TP_DIMENSION_X (float)16000 +#define J152F_TP_MIN_X -7456 +#define J152F_TP_MAX_X 7976 +#define J152F_TP_RES_X \ + ((J152F_TP_MAX_X - J152F_TP_MIN_X) / (J152F_TP_DIMENSION_X / 100)) +#define J152F_TP_DIMENSION_Y (float)10000 +#define J152F_TP_MIN_Y -163 +#define J152F_TP_MAX_Y 9283 +#define J152F_TP_RES_Y \ + ((J152F_TP_MAX_Y - J152F_TP_MIN_Y) / (J152F_TP_DIMENSION_Y / 100)) + +/* These are fallback values, since the real values will be queried from the device. */ +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define T2_TOUCHPAD_ENTRY(model) \ + { USB_DEVICE_ID_APPLE_WELLSPRINGT2_##model, model##_TP_MIN_X, model##_TP_MIN_Y, \ +model##_TP_MAX_X, model##_TP_MAX_Y, model##_TP_RES_X, model##_TP_RES_Y } + +#define INTERNAL_TP_MAX_FINGER_ORIENTATION 16384 + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. * @quirks: Currently unused. + * @query_dimensions: Whether to query and update dimensions on first open * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. + * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. + * @input_ops: Input ops based on device type. */ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; + bool query_dimensions; int ntouches; int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -143,14 +275,187 @@ struct magicmouse_sc { u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static int magicmouse_enable_multitouch(struct hid_device *hdev) +{ + const u8 *feature; + const u8 feature_mt[] = { 0xD7, 0x01 }; + const u8 feature_mt_mouse2[] = { 0xF1, 0x02, 0x01 }; + const u8 feature_mt_trackpad2_usb[] = { 0x02, 0x01 }; + const u8 feature_mt_trackpad2_bt[] = { 0xF1, 0x02, 0x01 }; + u8 *buf; + int ret; + int feature_size; + + switch (hdev->bus) { + case BUS_SPI: + case BUS_HOST: + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + break; + default: + switch (hdev->product) { + case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: + case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: + switch (hdev->vendor) { + case BT_VENDOR_ID_APPLE: + feature_size = sizeof(feature_mt_trackpad2_bt); + feature = feature_mt_trackpad2_bt; + break; + default: /* USB_VENDOR_ID_APPLE */ + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + } + break; + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + break; + case USB_DEVICE_ID_APPLE_MAGICMOUSE2: + case USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC: + feature_size = sizeof(feature_mt_mouse2); + feature = feature_mt_mouse2; + break; + default: + feature_size = sizeof(feature_mt); + feature = feature_mt; + } + } + + buf = kmemdup(feature, feature_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = hid_hw_raw_request(hdev, buf[0], buf, feature_size, + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + kfree(buf); + return ret; +} + +static void magicmouse_enable_mt_work(struct work_struct *work) +{ + struct magicmouse_sc *msc = + container_of(work, struct magicmouse_sc, work.work); + int ret; + + ret = magicmouse_enable_multitouch(msc->hdev); + if (ret < 0) + hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + * + * MTP devices do not need this. + */ + if (hdev->bus != BUS_HOST) { + ret = magicmouse_enable_multitouch(hdev); + if (ret == -EIO && hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + return 0; + } + if (ret < 0) + hid_err(hdev, "unable to request touch data (%d)\n", ret); + } + /* + * MT enable is usually not required after the first time, so don't + * consider it fatal. + */ + + /* + * For Apple Silicon trackpads, we want to query the dimensions on + * device open. This is because doing so requires the firmware, but + * we don't want to force a firmware load until the device is opened + * for the first time. So do that here and update the input properties + * just in time before userspace queries them. + */ + if (msc->query_dimensions) { + struct input_dev *input = msc->input; + u8 buf[32]; + struct { + __le32 width; + __le32 height; + __le16 min_x; + __le16 min_y; + __le16 max_x; + __le16 max_y; + } dim; + uint32_t x_span, y_span; + + ret = hid_hw_raw_request(hdev, SENSOR_DIMENSIONS_REPORT_ID, buf, sizeof(buf), HID_FEATURE_REPORT, HID_REQ_GET_REPORT); + if (ret < (int)(1 + sizeof(dim))) { + hid_err(hdev, "unable to request dimensions (%d)\n", ret); + return ret; + } + + memcpy(&dim, buf + 1, sizeof(dim)); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, + le16_to_int(dim.min_x), le16_to_int(dim.max_x), 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, + -le16_to_int(dim.max_y), -le16_to_int(dim.min_y), 0, 0); + x_span = le16_to_int(dim.max_x) - le16_to_int(dim.min_x); + y_span = le16_to_int(dim.max_y) - le16_to_int(dim.min_y); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, 100 * x_span / le32_to_cpu(dim.width) ); + input_abs_set_res(input, ABS_MT_POSITION_Y, 100 * y_span / le32_to_cpu(dim.height) ); + + /* copy info, as input_mt_init_slots() does */ + dev->absinfo[ABS_X] = dev->absinfo[ABS_MT_POSITION_X]; + dev->absinfo[ABS_Y] = dev->absinfo[ABS_MT_POSITION_Y]; + + msc->query_dimensions = false; + } + + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -192,7 +497,7 @@ static void magicmouse_emit_buttons(struct magicmouse_sc *msc, int state) } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -256,8 +561,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small @@ -387,6 +692,14 @@ static int magicmouse_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; @@ -518,6 +831,191 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + * @crc16: on last finger: crc over the whole message struct + * (i.e. message header + this struct) minus the last + * @crc16 field; unknown on all other fingers. + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * vendor trackpad report + * + * @num_fingers: the number of fingers being reported in @fingers + * @buttons: same as HID buttons + */ +struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; +}; + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + INTERNAL_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_mtp(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != TRACKPAD2_USB_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + +static int magicmouse_raw_event_t2(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] != TRACKPAD2_USB_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -536,7 +1034,17 @@ static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -615,7 +1123,7 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, @@ -695,6 +1203,171 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + return 0; +} + +struct magicmouse_t2_properties { + u32 id; + int min_x; + int min_y; + int max_x; + int max_y; + int res_x; + int res_y; +}; + +static const struct magicmouse_t2_properties magicmouse_t2_configs[] = { + T2_TOUCHPAD_ENTRY(J140K), + T2_TOUCHPAD_ENTRY(J132), + T2_TOUCHPAD_ENTRY(J680), + T2_TOUCHPAD_ENTRY(J680_ALT), + T2_TOUCHPAD_ENTRY(J213), + T2_TOUCHPAD_ENTRY(J214K), + T2_TOUCHPAD_ENTRY(J223), + T2_TOUCHPAD_ENTRY(J230K), + T2_TOUCHPAD_ENTRY(J152F), +}; + +static int magicmouse_setup_input_int_tpd(struct input_dev *input, + struct hid_device *hdev, int min_x, int min_y, + int max_x, int max_y, int res_x, int res_y, + bool query_dimensions) +{ + int error; + int mt_flags = 0; + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -INTERNAL_TP_MAX_FINGER_ORIENTATION, + INTERNAL_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, min_x, max_x, 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -max_y, -min_y, 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, res_x); + input_abs_set_res(input, ABS_MT_POSITION_Y, res_y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + * + * This also takes care of the dimensions query. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + msc->query_dimensions = query_dimensions; + + return 0; +} + +static int magicmouse_setup_input_mtp(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_int_tpd(input, hdev, J314_TP_MIN_X, + J314_TP_MIN_Y, J314_TP_MAX_X, + J314_TP_MAX_Y, J314_TP_RES_X, + J314_TP_RES_Y, true); + if (ret) + return ret; + + return 0; +} + +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_int_tpd(input, hdev, J314_TP_MIN_X, + J314_TP_MIN_Y, J314_TP_MAX_X, + J314_TP_MAX_Y, J314_TP_RES_X, + J314_TP_RES_Y, true); + if (ret) + return ret; + + return 0; +} + +static int magicmouse_setup_input_t2(struct input_dev *input, + struct hid_device *hdev) +{ + int min_x, min_y, max_x, max_y, res_x, res_y; + + for (size_t i = 0; i < ARRAY_SIZE(magicmouse_t2_configs); i++) { + if (magicmouse_t2_configs[i].id == hdev->product) { + min_x = magicmouse_t2_configs[i].min_x; + min_y = magicmouse_t2_configs[i].min_y; + max_x = magicmouse_t2_configs[i].max_x; + max_y = magicmouse_t2_configs[i].max_y; + res_x = magicmouse_t2_configs[i].res_x; + res_y = magicmouse_t2_configs[i].res_y; + } + } + + int ret = magicmouse_setup_input_int_tpd(input, hdev, min_x, min_y, + max_x, max_y, res_x, res_y, false); + if (ret) + return ret; + return 0; } @@ -736,86 +1409,17 @@ static int magicmouse_input_configured(struct hid_device *hdev, return 0; } -static int magicmouse_enable_multitouch(struct hid_device *hdev) -{ - const u8 *feature; - const u8 feature_mt[] = { 0xD7, 0x01 }; - const u8 feature_mt_mouse2[] = { 0xF1, 0x02, 0x01 }; - const u8 feature_mt_trackpad2_usb[] = { 0x02, 0x01 }; - const u8 feature_mt_trackpad2_bt[] = { 0xF1, 0x02, 0x01 }; - u8 *buf; - int ret; - int feature_size; - - switch (hdev->product) { - case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: - case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: - switch (hdev->vendor) { - case BT_VENDOR_ID_APPLE: - feature_size = sizeof(feature_mt_trackpad2_bt); - feature = feature_mt_trackpad2_bt; - break; - default: /* USB_VENDOR_ID_APPLE */ - feature_size = sizeof(feature_mt_trackpad2_usb); - feature = feature_mt_trackpad2_usb; - } - break; - case USB_DEVICE_ID_APPLE_MAGICMOUSE2: - case USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC: - feature_size = sizeof(feature_mt_mouse2); - feature = feature_mt_mouse2; - break; - default: - feature_size = sizeof(feature_mt); - feature = feature_mt; - } - - buf = kmemdup(feature, feature_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - ret = hid_hw_raw_request(hdev, buf[0], buf, feature_size, - HID_FEATURE_REPORT, HID_REQ_SET_REPORT); - kfree(buf); - return ret; -} - -static void magicmouse_enable_mt_work(struct work_struct *work) -{ - struct magicmouse_sc *msc = - container_of(work, struct magicmouse_sc, work.work); - int ret; - - ret = magicmouse_enable_multitouch(msc->hdev); - if (ret < 0) - hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); -} - -static bool is_usb_magicmouse2(__u32 vendor, __u32 product) -{ - if (vendor != USB_VENDOR_ID_APPLE) - return false; - return product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC; -} - -static bool is_usb_magictrackpad2(__u32 vendor, __u32 product) -{ - if (vendor != USB_VENDOR_ID_APPLE) - return false; - return product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || - product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC; -} - static int magicmouse_fetch_battery(struct hid_device *hdev) { #ifdef CONFIG_HID_BATTERY_STRENGTH struct hid_report_enum *report_enum; struct hid_report *report; - if (!hdev->battery || - (!is_usb_magicmouse2(hdev->vendor, hdev->product) && - !is_usb_magictrackpad2(hdev->vendor, hdev->product))) + if (!hdev->battery || hdev->vendor != USB_VENDOR_ID_APPLE || + (hdev->product != USB_DEVICE_ID_APPLE_MAGICMOUSE2 && + hdev->product != USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC && + hdev->product != USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 && + hdev->product != USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC)) return -1; report_enum = &hdev->report_enum[hdev->battery_report_type]; @@ -852,12 +1456,62 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + + switch (id->product) { + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + if (hdev->type != HID_TYPE_USBMOUSE) + return -ENODEV; + break; + } + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + switch (id->bus) { + case BUS_HOST: + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; + break; + case BUS_SPI: + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + break; + default: + switch (id->product) { + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + msc->input_ops.raw_event = magicmouse_raw_event_t2; + msc->input_ops.setup_input = magicmouse_setup_input_t2; + break; + default: + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } + } + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); @@ -877,17 +1531,17 @@ static int magicmouse_probe(struct hid_device *hdev, return ret; } - if (is_usb_magicmouse2(id->vendor, id->product) || - is_usb_magictrackpad2(id->vendor, id->product)) { - timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0); - mod_timer(&msc->battery_timer, - jiffies + msecs_to_jiffies(USB_BATTERY_TIMEOUT_MS)); - magicmouse_fetch_battery(hdev); - } - - if (is_usb_magicmouse2(id->vendor, id->product) || - (is_usb_magictrackpad2(id->vendor, id->product) && - hdev->type != HID_TYPE_USBMOUSE)) + timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0); + mod_timer(&msc->battery_timer, + jiffies + msecs_to_jiffies(USB_BATTERY_TIMEOUT_MS)); + magicmouse_fetch_battery(hdev); + + if (id->vendor == USB_VENDOR_ID_APPLE && + (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || + id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC || + ((id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || + id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC) && + hdev->type != HID_TYPE_USBMOUSE))) return 0; if (!msc->input) { @@ -916,11 +1570,29 @@ static int magicmouse_probe(struct hid_device *hdev, TRACKPAD2_USB_REPORT_ID, 0); } break; - default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD_REPORT_ID, 0); + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: report = hid_register_report(hdev, HID_INPUT_REPORT, - DOUBLE_REPORT_ID, 0); + TRACKPAD2_USB_REPORT_ID, 0); + break; + default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + } else if (id->bus == BUS_HOST) { + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + } else { + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD_REPORT_ID, 0); + report = hid_register_report(hdev, HID_INPUT_REPORT, + DOUBLE_REPORT_ID, 0); + } } if (!report) { @@ -930,6 +1602,10 @@ static int magicmouse_probe(struct hid_device *hdev, } report->size = 6; + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + /* * Some devices repond with 'invalid report id' when feature * report switching it into multitouch mode is sent to it. @@ -950,10 +1626,7 @@ static int magicmouse_probe(struct hid_device *hdev, return 0; err_stop_hw: - if (is_usb_magicmouse2(id->vendor, id->product) || - is_usb_magictrackpad2(id->vendor, id->product)) - timer_delete_sync(&msc->battery_timer); - + timer_delete_sync(&msc->battery_timer); hid_hw_stop(hdev); return ret; } @@ -964,9 +1637,7 @@ static void magicmouse_remove(struct hid_device *hdev) if (msc) { cancel_delayed_work_sync(&msc->work); - if (is_usb_magicmouse2(hdev->vendor, hdev->product) || - is_usb_magictrackpad2(hdev->vendor, hdev->product)) - timer_delete_sync(&msc->battery_timer); + timer_delete_sync(&msc->battery_timer); } hid_hw_stop(hdev); @@ -983,8 +1654,11 @@ static const __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, * 0x05, 0x01, // Usage Page (Generic Desktop) 0 * 0x09, 0x02, // Usage (Mouse) 2 */ - if ((is_usb_magicmouse2(hdev->vendor, hdev->product) || - is_usb_magictrackpad2(hdev->vendor, hdev->product)) && + if (hdev->vendor == USB_VENDOR_ID_APPLE && + (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || + hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC || + hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || + hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC) && *rsize == 83 && rdesc[46] == 0x84 && rdesc[58] == 0x85) { hid_info(hdev, "fixing up magicmouse battery report descriptor\n"); @@ -1023,10 +1697,42 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + if (hdev->bus == BUS_SPI) + return magicmouse_enable_multitouch(hdev); + + return 0; +} +#endif + static struct hid_driver magicmouse_driver = { .name = "magicmouse", .id_table = magic_mice, @@ -1037,6 +1743,10 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif + }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index a1c54ffe02b4..294516a8f541 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -73,6 +73,7 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_FORCE_MULTI_INPUT BIT(20) #define MT_QUIRK_DISABLE_WAKEUP BIT(21) #define MT_QUIRK_ORIENTATION_INVERT BIT(22) +#define MT_QUIRK_APPLE_TOUCHBAR BIT(23) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -220,6 +221,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_GOOGLE 0x0111 #define MT_CLS_RAZER_BLADE_STEALTH 0x0112 #define MT_CLS_SMART_TECH 0x0113 +#define MT_CLS_APPLE_TOUCHBAR 0x0114 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -405,6 +407,12 @@ static const struct mt_class mt_classes[] = { MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_SEPARATE_APP_REPORT, }, + { .name = MT_CLS_APPLE_TOUCHBAR, + .quirks = MT_QUIRK_HOVERING | + MT_QUIRK_SLOT_IS_CONTACTID_MINUS_ONE | + MT_QUIRK_APPLE_TOUCHBAR, + .maxcontacts = 11, + }, { .name = MT_CLS_SIS, .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | MT_QUIRK_ALWAYS_VALID | @@ -625,6 +633,7 @@ static struct mt_application *mt_find_application(struct mt_device *td, static struct mt_report_data *mt_allocate_report_data(struct mt_device *td, struct hid_report *report) { + struct mt_class *cls = &td->mtclass; struct mt_report_data *rdata; struct hid_field *field; int r, n; @@ -649,7 +658,11 @@ static struct mt_report_data *mt_allocate_report_data(struct mt_device *td, if (field->logical == HID_DG_FINGER || td->hdev->group != HID_GROUP_MULTITOUCH_WIN_8) { for (n = 0; n < field->report_count; n++) { - if (field->usage[n].hid == HID_DG_CONTACTID) { + unsigned int hid = field->usage[n].hid; + + if (hid == HID_DG_CONTACTID || + (cls->quirks & MT_QUIRK_APPLE_TOUCHBAR && + hid == HID_DG_TRANSDUCER_INDEX)) { rdata->is_mt_collection = true; break; } @@ -821,12 +834,31 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, MT_STORE_FIELD(confidence_state); return 1; + case HID_DG_TOUCH: + /* + * Legacy devices use TIPSWITCH and not TOUCH. + * One special case here is of the Apple Touch Bars. + * In these devices, the tip state is contained in + * fields with the HID_DG_TOUCH usage. + * Let's just ignore this field for other devices. + */ + if (!(cls->quirks & MT_QUIRK_APPLE_TOUCHBAR)) + return -1; + fallthrough; case HID_DG_TIPSWITCH: if (field->application != HID_GD_SYSTEM_MULTIAXIS) input_set_capability(hi->input, EV_KEY, BTN_TOUCH); MT_STORE_FIELD(tip_state); return 1; + case HID_DG_TRANSDUCER_INDEX: + /* + * Contact ID in case of Apple Touch Bars is contained + * in fields with HID_DG_TRANSDUCER_INDEX usage. + */ + if (!(cls->quirks & MT_QUIRK_APPLE_TOUCHBAR)) + return 0; + fallthrough; case HID_DG_CONTACTID: MT_STORE_FIELD(contactid); app->touches_by_report++; @@ -883,10 +915,6 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, case HID_DG_CONTACTMAX: /* contact max are global to the report */ return -1; - case HID_DG_TOUCH: - /* Legacy devices use TIPSWITCH and not TOUCH. - * Let's just ignore this field. */ - return -1; } /* let hid-input decide for the others */ return 0; @@ -1314,6 +1342,13 @@ static int mt_touch_input_configured(struct hid_device *hdev, struct input_dev *input = hi->input; int ret; + /* + * HID_DG_CONTACTMAX field is not present on Apple Touch Bars, + * but the maximum contact count is greater than the default. + */ + if (cls->quirks & MT_QUIRK_APPLE_TOUCHBAR && cls->maxcontacts) + td->maxcontacts = cls->maxcontacts; + if (!td->maxcontacts) td->maxcontacts = MT_DEFAULT_MAXCONTACT; @@ -1321,6 +1356,13 @@ static int mt_touch_input_configured(struct hid_device *hdev, if (td->serial_maybe) mt_post_parse_default_settings(td, app); + /* + * The application for Apple Touch Bars is HID_DG_TOUCHPAD, + * but these devices are direct. + */ + if (cls->quirks & MT_QUIRK_APPLE_TOUCHBAR) + app->mt_flags |= INPUT_MT_DIRECT; + if (cls->is_indirect) app->mt_flags |= INPUT_MT_POINTER; @@ -1823,6 +1865,11 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id) if (ret != 0) return ret; + if (mtclass->name == MT_CLS_APPLE_TOUCHBAR && + !hid_find_field(hdev, HID_INPUT_REPORT, + HID_DG_TOUCHPAD, HID_DG_TRANSDUCER_INDEX)) + return -ENODEV; + if (mtclass->quirks & MT_QUIRK_FIX_CONST_CONTACT_ID) mt_fix_const_fields(hdev, HID_DG_CONTACTID); @@ -2320,6 +2367,11 @@ static const struct hid_device_id mt_devices[] = { MT_USB_DEVICE(USB_VENDOR_ID_XIROKU, USB_DEVICE_ID_XIROKU_CSR2) }, + /* Apple Touch Bar */ + { .driver_data = MT_CLS_APPLE_TOUCHBAR, + HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, + /* Google MT devices */ { .driver_data = MT_CLS_GOOGLE, HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_GOOGLE, diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 9bf9ce8dc803..ff11f1ad344d 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -314,6 +314,7 @@ static const struct hid_device_id hid_have_special_driver[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223) }, @@ -973,14 +974,6 @@ static const struct hid_device_id hid_mouse_ignore_list[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ISO) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) }, { } diff --git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig new file mode 100644 index 000000000000..8e37f0fec28a --- /dev/null +++ b/drivers/hid/spi-hid/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + default ARCH_APPLE + depends on SPI && INPUT && OF + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + default y if SPI_HID_APPLE_OF=y + default m if SPI_HID_APPLE_OF=m + select HID + select CRC16 diff --git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile new file mode 100644 index 000000000000..f276ee12cb94 --- /dev/null +++ b/drivers/hid/spi-hid/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c new file mode 100644 index 000000000000..1f8fa64d6d86 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -0,0 +1,1194 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + u8 reply_buf[SPIHID_DESC_MAX]; + u32 reply_len; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @msgid: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + int ret; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + if (rtype != HID_FEATURE_REPORT) + return -EINVAL; + + idev->reply_len = 0; + ret = spihid_apple_request(spihid, idev->id, 0x32, reportnum, 0x00, len, NULL, 0); + if (ret < 0) + return ret; + + ret = wait_event_interruptible_timeout(spihid->wait, idev->reply_len, + SPIHID_DEF_WAIT); + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) { + dev_err(&spihid->spidev->dev, "waiting for get report failed: %d", ret); + return ret; + } + memcpy(buf, idev->reply_buf, max_t(size_t, len, idev->reply_len)); + return idev->reply_len; + + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, + .max_buffer_size = SPIHID_MAX_INPUT_REPORT_SIZE, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_iface_get_report(struct spihid_apple *spihid, + u32 device, u8 report, + u8 *payload, size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, device); + + if (!iface) + return false; + + if (len > sizeof(iface->reply_buf) || len < 1) + return false; + + memcpy(iface->reply_buf, payload, len); + iface->reply_len = len; + + wake_up_interruptible(&spihid->wait); + + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + if (hdr->unknown0 == 0x32) { + return spihid_process_iface_get_report(spihid, device, hdr->unknown1, payload, len); + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + size_t payload_len; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + payload_len = le16_to_cpu(hdr->length); + + if (payload_len == 0 || + (payload_len + sizeof(struct spihid_msg_hdr) + 2) > length) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, payload_len); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, device, hdr, payload, + payload_len); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + char *suffix; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + /* + * Use 'Apple SPI Keyboard' and 'Apple SPI Trackpad' as input device + * names. The device names need to be distinct since at least Kwin uses + * the tripple Vendor ID, Product ID, Name to identify devices. + */ + snprintf(hid->name, sizeof(hid->name), "Apple SPI %s", iface->name); + // strip ' / Boot' suffix from the name + suffix = strstr(hid->name, " / Boot"); + if (suffix) + suffix[0] = '\0'; + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* + * allocate SPI buffers + * Overallocate the receice buffer since it passed directly into + * hid_input_report / hid_report_raw_event. The later expects the buffer + * to be HID_MAX_BUFFER_SIZE (16k) or hid_ll_driver.max_buffer_size if + * set. + */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c new file mode 100644 index 000000000000..3f87b299351d --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -0,0 +1,151 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + .pm = &spihid_apple_core_pm, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h new file mode 100644 index 000000000000..9abecd1ba780 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +extern const struct dev_pm_ops spihid_apple_core_pm; + +#endif /* SPI_HID_APPLE_H */ diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c index fc6d6a9053ce..698f44794453 100644 --- a/drivers/hwmon/applesmc.c +++ b/drivers/hwmon/applesmc.c @@ -6,6 +6,7 @@ * * Copyright (C) 2007 Nicolas Boichat * Copyright (C) 2010 Henrik Rydberg + * Copyright (C) 2019 Paul Pawlowski * * Based on hdaps.c driver: * Copyright (C) 2005 Robert Love @@ -18,7 +19,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include -#include +#include #include #include #include @@ -35,12 +36,24 @@ #include /* data port used by Apple SMC */ -#define APPLESMC_DATA_PORT 0x300 +#define APPLESMC_DATA_PORT 0 /* command/status port used by Apple SMC */ -#define APPLESMC_CMD_PORT 0x304 +#define APPLESMC_CMD_PORT 4 #define APPLESMC_NR_PORTS 32 /* 0x300-0x31f */ +#define APPLESMC_IOMEM_KEY_DATA 0 +#define APPLESMC_IOMEM_KEY_STATUS 0x4005 +#define APPLESMC_IOMEM_KEY_NAME 0x78 +#define APPLESMC_IOMEM_KEY_DATA_LEN 0x7D +#define APPLESMC_IOMEM_KEY_SMC_ID 0x7E +#define APPLESMC_IOMEM_KEY_CMD 0x7F +#define APPLESMC_IOMEM_MIN_SIZE 0x4006 + +#define APPLESMC_IOMEM_KEY_TYPE_CODE 0 +#define APPLESMC_IOMEM_KEY_TYPE_DATA_LEN 5 +#define APPLESMC_IOMEM_KEY_TYPE_FLAGS 6 + #define APPLESMC_MAX_DATA_LENGTH 32 /* Apple SMC status bits */ @@ -74,6 +87,7 @@ #define FAN_ID_FMT "F%dID" /* r-o char[16] */ #define TEMP_SENSOR_TYPE "sp78" +#define FLOAT_TYPE "flt " /* List of keys used to read/write fan speeds */ static const char *const fan_speed_fmt[] = { @@ -83,6 +97,7 @@ static const char *const fan_speed_fmt[] = { "F%dSf", /* safe speed - not all models */ "F%dTg", /* target speed (manual: rw) */ }; +#define FAN_MANUAL_FMT "F%dMd" #define INIT_TIMEOUT_MSECS 5000 /* wait up to 5s for device init ... */ #define INIT_WAIT_MSECS 50 /* ... in 50ms increments */ @@ -119,7 +134,7 @@ struct applesmc_entry { }; /* Register lookup and registers common to all SMCs */ -static struct applesmc_registers { +struct applesmc_registers { struct mutex mutex; /* register read/write mutex */ unsigned int key_count; /* number of SMC registers */ unsigned int fan_count; /* number of fans */ @@ -133,26 +148,38 @@ static struct applesmc_registers { bool init_complete; /* true when fully initialized */ struct applesmc_entry *cache; /* cached key entries */ const char **index; /* temperature key index */ -} smcreg = { - .mutex = __MUTEX_INITIALIZER(smcreg.mutex), }; -static const int debug; -static struct platform_device *pdev; -static s16 rest_x; -static s16 rest_y; -static u8 backlight_state[2]; +struct applesmc_device { + struct acpi_device *dev; + struct device *ldev; + struct applesmc_registers reg; -static struct device *hwmon_dev; -static struct input_dev *applesmc_idev; + bool port_base_set, iomem_base_set; + u16 port_base; + u8 *__iomem iomem_base; + u32 iomem_base_addr, iomem_base_size; -/* - * Last index written to key_at_index sysfs file, and value to use for all other - * key_at_index_* sysfs files. - */ -static unsigned int key_at_index; + s16 rest_x; + s16 rest_y; + + u8 backlight_state[2]; + + struct device *hwmon_dev; + struct input_dev *idev; + + /* + * Last index written to key_at_index sysfs file, and value to use for all other + * key_at_index_* sysfs files. + */ + unsigned int key_at_index; -static struct workqueue_struct *applesmc_led_wq; + struct workqueue_struct *backlight_wq; + struct work_struct backlight_work; + struct led_classdev backlight_dev; +}; + +static const int debug; /* * Wait for specific status bits with a mask on the SMC. @@ -162,7 +189,7 @@ static struct workqueue_struct *applesmc_led_wq; * run out past 500ms. */ -static int wait_status(u8 val, u8 mask) +static int port_wait_status(struct applesmc_device *smc, u8 val, u8 mask) { u8 status; int us; @@ -170,7 +197,7 @@ static int wait_status(u8 val, u8 mask) us = APPLESMC_MIN_WAIT; for (i = 0; i < 24 ; i++) { - status = inb(APPLESMC_CMD_PORT); + status = inb(smc->port_base + APPLESMC_CMD_PORT); if ((status & mask) == val) return 0; usleep_range(us, us * 2); @@ -180,13 +207,13 @@ static int wait_status(u8 val, u8 mask) return -EIO; } -/* send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ +/* port_send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ -static int send_byte(u8 cmd, u16 port) +static int port_send_byte(struct applesmc_device *smc, u8 cmd, u16 port) { int status; - status = wait_status(0, SMC_STATUS_IB_CLOSED); + status = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); if (status) return status; /* @@ -195,24 +222,25 @@ static int send_byte(u8 cmd, u16 port) * this extra read may not happen if status returns both * simultaneously and this would appear to be required. */ - status = wait_status(SMC_STATUS_BUSY, SMC_STATUS_BUSY); + status = port_wait_status(smc, SMC_STATUS_BUSY, SMC_STATUS_BUSY); if (status) return status; - outb(cmd, port); + outb(cmd, smc->port_base + port); return 0; } -/* send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ +/* port_send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ -static int send_command(u8 cmd) +static int port_send_command(struct applesmc_device *smc, u8 cmd) { int ret; - ret = wait_status(0, SMC_STATUS_IB_CLOSED); + ret = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); if (ret) return ret; - outb(cmd, APPLESMC_CMD_PORT); + + outb(cmd, smc->port_base + APPLESMC_CMD_PORT); return 0; } @@ -222,110 +250,304 @@ static int send_command(u8 cmd) * If busy is stuck high after the command then the SMC is jammed. */ -static int smc_sane(void) +static int port_smc_sane(struct applesmc_device *smc) { int ret; - ret = wait_status(0, SMC_STATUS_BUSY); + ret = port_wait_status(smc, 0, SMC_STATUS_BUSY); if (!ret) return ret; - ret = send_command(APPLESMC_READ_CMD); + ret = port_send_command(smc, APPLESMC_READ_CMD); if (ret) return ret; - return wait_status(0, SMC_STATUS_BUSY); + return port_wait_status(smc, 0, SMC_STATUS_BUSY); } -static int send_argument(const char *key) +static int port_send_argument(struct applesmc_device *smc, const char *key) { int i; for (i = 0; i < 4; i++) - if (send_byte(key[i], APPLESMC_DATA_PORT)) + if (port_send_byte(smc, key[i], APPLESMC_DATA_PORT)) return -EIO; return 0; } -static int read_smc(u8 cmd, const char *key, u8 *buffer, u8 len) +static int port_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, + u8 *buffer, u8 len) { u8 status, data = 0; int i; int ret; - ret = smc_sane(); + ret = port_smc_sane(smc); if (ret) return ret; - if (send_command(cmd) || send_argument(key)) { + if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { pr_warn("%.4s: read arg fail\n", key); return -EIO; } /* This has no effect on newer (2012) SMCs */ - if (send_byte(len, APPLESMC_DATA_PORT)) { + if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { pr_warn("%.4s: read len fail\n", key); return -EIO; } for (i = 0; i < len; i++) { - if (wait_status(SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, + if (port_wait_status(smc, + SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY)) { pr_warn("%.4s: read data[%d] fail\n", key, i); return -EIO; } - buffer[i] = inb(APPLESMC_DATA_PORT); + buffer[i] = inb(smc->port_base + APPLESMC_DATA_PORT); } /* Read the data port until bit0 is cleared */ for (i = 0; i < 16; i++) { udelay(APPLESMC_MIN_WAIT); - status = inb(APPLESMC_CMD_PORT); + status = inb(smc->port_base + APPLESMC_CMD_PORT); if (!(status & SMC_STATUS_AWAITING_DATA)) break; - data = inb(APPLESMC_DATA_PORT); + data = inb(smc->port_base + APPLESMC_DATA_PORT); } if (i) pr_warn("flushed %d bytes, last value is: %d\n", i, data); - return wait_status(0, SMC_STATUS_BUSY); + return port_wait_status(smc, 0, SMC_STATUS_BUSY); } -static int write_smc(u8 cmd, const char *key, const u8 *buffer, u8 len) +static int port_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, + const u8 *buffer, u8 len) { int i; int ret; - ret = smc_sane(); + ret = port_smc_sane(smc); if (ret) return ret; - if (send_command(cmd) || send_argument(key)) { + if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { pr_warn("%s: write arg fail\n", key); return -EIO; } - if (send_byte(len, APPLESMC_DATA_PORT)) { + if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { pr_warn("%.4s: write len fail\n", key); return -EIO; } for (i = 0; i < len; i++) { - if (send_byte(buffer[i], APPLESMC_DATA_PORT)) { + if (port_send_byte(smc, buffer[i], APPLESMC_DATA_PORT)) { pr_warn("%s: write data fail\n", key); return -EIO; } } - return wait_status(0, SMC_STATUS_BUSY); + return port_wait_status(smc, 0, SMC_STATUS_BUSY); } -static int read_register_count(unsigned int *count) +static int port_get_smc_key_info(struct applesmc_device *smc, + const char *key, struct applesmc_entry *info) { - __be32 be; int ret; + u8 raw[6]; - ret = read_smc(APPLESMC_READ_CMD, KEY_COUNT_KEY, (u8 *)&be, 4); + ret = port_read_smc(smc, APPLESMC_GET_KEY_TYPE_CMD, key, raw, 6); if (ret) return ret; + info->len = raw[0]; + memcpy(info->type, &raw[1], 4); + info->flags = raw[5]; + return 0; +} + + +/* + * MMIO based communication. + * TODO: Use updated mechanism for cmd timeout/retry + */ + +static void iomem_clear_status(struct applesmc_device *smc) +{ + if (ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS)) + iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); +} + +static int iomem_wait_read(struct applesmc_device *smc) +{ + u8 status; + int us; + int i; + + us = APPLESMC_MIN_WAIT; + for (i = 0; i < 24 ; i++) { + status = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); + if (status & 0x20) + return 0; + usleep_range(us, us * 2); + if (i > 9) + us <<= 1; + } + + dev_warn(smc->ldev, "%s... timeout\n", __func__); + return -EIO; +} + +static int iomem_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, + u8 *buffer, u8 len) +{ + u8 err, remote_len; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "read_smc_mmio(%x %8x/%.4s) failed: %u\n", + cmd, key_int, key, err); + return -EIO; + } + + if (cmd == APPLESMC_READ_CMD) { + remote_len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); + if (remote_len != len) { + dev_warn(smc->ldev, + "read_smc_mmio(%x %8x/%.4s) failed: buffer length mismatch (remote = %u, requested = %u)\n", + cmd, key_int, key, remote_len, len); + return -EINVAL; + } + } else { + remote_len = len; + } + + memcpy_fromio(buffer, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, + remote_len); + + dev_dbg(smc->ldev, "read_smc_mmio(%x %8x/%.4s): buflen=%u reslen=%u\n", + cmd, key_int, key, len, remote_len); + print_hex_dump_bytes("read_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, remote_len); + return 0; +} + +static int iomem_get_smc_key_type(struct applesmc_device *smc, const char *key, + struct applesmc_entry *e) +{ + u8 err; + u8 cmd = APPLESMC_GET_KEY_TYPE_CMD; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "get_smc_key_type_mmio(%.4s) failed: %u\n", key, err); + return -EIO; + } + + e->len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_DATA_LEN); + *((uint32_t *) e->type) = ioread32( + smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_CODE); + e->flags = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_FLAGS); + + dev_dbg(smc->ldev, "get_smc_key_type_mmio(%.4s): len=%u type=%.4s flags=%x\n", + key, e->len, e->type, e->flags); + return 0; +} + +static int iomem_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, + const u8 *buffer, u8 len) +{ + u8 err; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + memcpy_toio(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, buffer, len); + iowrite32(len, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); + iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "write_smc_mmio(%x %.4s) failed: %u\n", cmd, key, err); + print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); + return -EIO; + } + + dev_dbg(smc->ldev, "write_smc_mmio(%x %.4s): buflen=%u\n", cmd, key, len); + print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); + return 0; +} + + +static int read_smc(struct applesmc_device *smc, const char *key, + u8 *buffer, u8 len) +{ + if (smc->iomem_base_set) + return iomem_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); + else + return port_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); +} + +static int write_smc(struct applesmc_device *smc, const char *key, + const u8 *buffer, u8 len) +{ + if (smc->iomem_base_set) + return iomem_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); + else + return port_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); +} + +static int get_smc_key_by_index(struct applesmc_device *smc, + unsigned int index, char *key) +{ + __be32 be; + + be = cpu_to_be32(index); + if (smc->iomem_base_set) + return iomem_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, + (const char *) &be, (u8 *) key, 4); + else + return port_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, + (const char *) &be, (u8 *) key, 4); +} + +static int get_smc_key_info(struct applesmc_device *smc, const char *key, + struct applesmc_entry *info) +{ + if (smc->iomem_base_set) + return iomem_get_smc_key_type(smc, key, info); + else + return port_get_smc_key_info(smc, key, info); +} + +static int read_register_count(struct applesmc_device *smc, + unsigned int *count) +{ + __be32 be; + int ret; + + ret = read_smc(smc, KEY_COUNT_KEY, (u8 *)&be, 4); + if (ret < 0) + return ret; *count = be32_to_cpu(be); return 0; @@ -338,76 +560,73 @@ static int read_register_count(unsigned int *count) * All functions below are concurrency safe - callers should NOT hold lock. */ -static int applesmc_read_entry(const struct applesmc_entry *entry, - u8 *buf, u8 len) +static int applesmc_read_entry(struct applesmc_device *smc, + const struct applesmc_entry *entry, u8 *buf, u8 len) { int ret; if (entry->len != len) return -EINVAL; - mutex_lock(&smcreg.mutex); - ret = read_smc(APPLESMC_READ_CMD, entry->key, buf, len); - mutex_unlock(&smcreg.mutex); + mutex_lock(&smc->reg.mutex); + ret = read_smc(smc, entry->key, buf, len); + mutex_unlock(&smc->reg.mutex); return ret; } -static int applesmc_write_entry(const struct applesmc_entry *entry, - const u8 *buf, u8 len) +static int applesmc_write_entry(struct applesmc_device *smc, + const struct applesmc_entry *entry, const u8 *buf, u8 len) { int ret; if (entry->len != len) return -EINVAL; - mutex_lock(&smcreg.mutex); - ret = write_smc(APPLESMC_WRITE_CMD, entry->key, buf, len); - mutex_unlock(&smcreg.mutex); + mutex_lock(&smc->reg.mutex); + ret = write_smc(smc, entry->key, buf, len); + mutex_unlock(&smc->reg.mutex); return ret; } -static const struct applesmc_entry *applesmc_get_entry_by_index(int index) +static const struct applesmc_entry *applesmc_get_entry_by_index( + struct applesmc_device *smc, int index) { - struct applesmc_entry *cache = &smcreg.cache[index]; - u8 key[4], info[6]; - __be32 be; + struct applesmc_entry *cache = &smc->reg.cache[index]; + char key[4]; int ret = 0; if (cache->valid) return cache; - mutex_lock(&smcreg.mutex); + mutex_lock(&smc->reg.mutex); if (cache->valid) goto out; - be = cpu_to_be32(index); - ret = read_smc(APPLESMC_GET_KEY_BY_INDEX_CMD, (u8 *)&be, key, 4); + ret = get_smc_key_by_index(smc, index, key); if (ret) goto out; - ret = read_smc(APPLESMC_GET_KEY_TYPE_CMD, key, info, 6); + memcpy(cache->key, key, 4); + + ret = get_smc_key_info(smc, key, cache); if (ret) goto out; - - memcpy(cache->key, key, 4); - cache->len = info[0]; - memcpy(cache->type, &info[1], 4); - cache->flags = info[5]; cache->valid = true; out: - mutex_unlock(&smcreg.mutex); + mutex_unlock(&smc->reg.mutex); if (ret) return ERR_PTR(ret); return cache; } -static int applesmc_get_lower_bound(unsigned int *lo, const char *key) +static int applesmc_get_lower_bound(struct applesmc_device *smc, + unsigned int *lo, const char *key) { - int begin = 0, end = smcreg.key_count; + int begin = 0, end = smc->reg.key_count; const struct applesmc_entry *entry; while (begin != end) { int middle = begin + (end - begin) / 2; - entry = applesmc_get_entry_by_index(middle); + entry = applesmc_get_entry_by_index(smc, middle); if (IS_ERR(entry)) { *lo = 0; return PTR_ERR(entry); @@ -422,16 +641,17 @@ static int applesmc_get_lower_bound(unsigned int *lo, const char *key) return 0; } -static int applesmc_get_upper_bound(unsigned int *hi, const char *key) +static int applesmc_get_upper_bound(struct applesmc_device *smc, + unsigned int *hi, const char *key) { - int begin = 0, end = smcreg.key_count; + int begin = 0, end = smc->reg.key_count; const struct applesmc_entry *entry; while (begin != end) { int middle = begin + (end - begin) / 2; - entry = applesmc_get_entry_by_index(middle); + entry = applesmc_get_entry_by_index(smc, middle); if (IS_ERR(entry)) { - *hi = smcreg.key_count; + *hi = smc->reg.key_count; return PTR_ERR(entry); } if (strcmp(key, entry->key) < 0) @@ -444,50 +664,54 @@ static int applesmc_get_upper_bound(unsigned int *hi, const char *key) return 0; } -static const struct applesmc_entry *applesmc_get_entry_by_key(const char *key) +static const struct applesmc_entry *applesmc_get_entry_by_key( + struct applesmc_device *smc, const char *key) { int begin, end; int ret; - ret = applesmc_get_lower_bound(&begin, key); + ret = applesmc_get_lower_bound(smc, &begin, key); if (ret) return ERR_PTR(ret); - ret = applesmc_get_upper_bound(&end, key); + ret = applesmc_get_upper_bound(smc, &end, key); if (ret) return ERR_PTR(ret); if (end - begin != 1) return ERR_PTR(-EINVAL); - return applesmc_get_entry_by_index(begin); + return applesmc_get_entry_by_index(smc, begin); } -static int applesmc_read_key(const char *key, u8 *buffer, u8 len) +static int applesmc_read_key(struct applesmc_device *smc, + const char *key, u8 *buffer, u8 len) { const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_key(key); + entry = applesmc_get_entry_by_key(smc, key); if (IS_ERR(entry)) return PTR_ERR(entry); - return applesmc_read_entry(entry, buffer, len); + return applesmc_read_entry(smc, entry, buffer, len); } -static int applesmc_write_key(const char *key, const u8 *buffer, u8 len) +static int applesmc_write_key(struct applesmc_device *smc, + const char *key, const u8 *buffer, u8 len) { const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_key(key); + entry = applesmc_get_entry_by_key(smc, key); if (IS_ERR(entry)) return PTR_ERR(entry); - return applesmc_write_entry(entry, buffer, len); + return applesmc_write_entry(smc, entry, buffer, len); } -static int applesmc_has_key(const char *key, bool *value) +static int applesmc_has_key(struct applesmc_device *smc, + const char *key, bool *value) { const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_key(key); + entry = applesmc_get_entry_by_key(smc, key); if (IS_ERR(entry) && PTR_ERR(entry) != -EINVAL) return PTR_ERR(entry); @@ -498,12 +722,13 @@ static int applesmc_has_key(const char *key, bool *value) /* * applesmc_read_s16 - Read 16-bit signed big endian register */ -static int applesmc_read_s16(const char *key, s16 *value) +static int applesmc_read_s16(struct applesmc_device *smc, + const char *key, s16 *value) { u8 buffer[2]; int ret; - ret = applesmc_read_key(key, buffer, 2); + ret = applesmc_read_key(smc, key, buffer, 2); if (ret) return ret; @@ -511,31 +736,68 @@ static int applesmc_read_s16(const char *key, s16 *value) return 0; } +/** + * applesmc_float_to_u32 - Retrieve the integral part of a float. + * This is needed because Apple made fans use float values in the T2. + * The fractional point is not significantly useful though, and the integral + * part can be easily extracted. + */ +static inline u32 applesmc_float_to_u32(u32 d) +{ + u8 sign = (u8) ((d >> 31) & 1); + s32 exp = (s32) ((d >> 23) & 0xff) - 0x7f; + u32 fr = d & ((1u << 23) - 1); + + if (sign || exp < 0) + return 0; + + return (u32) ((1u << exp) + (fr >> (23 - exp))); +} + +/** + * applesmc_u32_to_float - Convert an u32 into a float. + * See applesmc_float_to_u32 for a rationale. + */ +static inline u32 applesmc_u32_to_float(u32 d) +{ + u32 dc = d, bc = 0, exp; + + if (!d) + return 0; + + while (dc >>= 1) + ++bc; + exp = 0x7f + bc; + + return (u32) ((exp << 23) | + ((d << (23 - (exp - 0x7f))) & ((1u << 23) - 1))); +} /* * applesmc_device_init - initialize the accelerometer. Can sleep. */ -static void applesmc_device_init(void) +static void applesmc_device_init(struct applesmc_device *smc) { int total; u8 buffer[2]; - if (!smcreg.has_accelerometer) + if (!smc->reg.has_accelerometer) return; for (total = INIT_TIMEOUT_MSECS; total > 0; total -= INIT_WAIT_MSECS) { - if (!applesmc_read_key(MOTION_SENSOR_KEY, buffer, 2) && + if (!applesmc_read_key(smc, MOTION_SENSOR_KEY, buffer, 2) && (buffer[0] != 0x00 || buffer[1] != 0x00)) return; buffer[0] = 0xe0; buffer[1] = 0x00; - applesmc_write_key(MOTION_SENSOR_KEY, buffer, 2); + applesmc_write_key(smc, MOTION_SENSOR_KEY, buffer, 2); msleep(INIT_WAIT_MSECS); } pr_warn("failed to init the device\n"); } -static int applesmc_init_index(struct applesmc_registers *s) +static int applesmc_init_index(struct applesmc_device *smc, + struct applesmc_registers *s) { const struct applesmc_entry *entry; unsigned int i; @@ -548,7 +810,7 @@ static int applesmc_init_index(struct applesmc_registers *s) return -ENOMEM; for (i = s->temp_begin; i < s->temp_end; i++) { - entry = applesmc_get_entry_by_index(i); + entry = applesmc_get_entry_by_index(smc, i); if (IS_ERR(entry)) continue; if (strcmp(entry->type, TEMP_SENSOR_TYPE)) @@ -562,9 +824,9 @@ static int applesmc_init_index(struct applesmc_registers *s) /* * applesmc_init_smcreg_try - Try to initialize register cache. Idempotent. */ -static int applesmc_init_smcreg_try(void) +static int applesmc_init_smcreg_try(struct applesmc_device *smc) { - struct applesmc_registers *s = &smcreg; + struct applesmc_registers *s = &smc->reg; bool left_light_sensor = false, right_light_sensor = false; unsigned int count; u8 tmp[1]; @@ -573,7 +835,7 @@ static int applesmc_init_smcreg_try(void) if (s->init_complete) return 0; - ret = read_register_count(&count); + ret = read_register_count(smc, &count); if (ret) return ret; @@ -590,35 +852,35 @@ static int applesmc_init_smcreg_try(void) if (!s->cache) return -ENOMEM; - ret = applesmc_read_key(FANS_COUNT, tmp, 1); + ret = applesmc_read_key(smc, FANS_COUNT, tmp, 1); if (ret) return ret; s->fan_count = tmp[0]; if (s->fan_count > 10) s->fan_count = 10; - ret = applesmc_get_lower_bound(&s->temp_begin, "T"); + ret = applesmc_get_lower_bound(smc, &s->temp_begin, "T"); if (ret) return ret; - ret = applesmc_get_lower_bound(&s->temp_end, "U"); + ret = applesmc_get_lower_bound(smc, &s->temp_end, "U"); if (ret) return ret; s->temp_count = s->temp_end - s->temp_begin; - ret = applesmc_init_index(s); + ret = applesmc_init_index(smc, s); if (ret) return ret; - ret = applesmc_has_key(LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); + ret = applesmc_has_key(smc, LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); if (ret) return ret; - ret = applesmc_has_key(LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); + ret = applesmc_has_key(smc, LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); if (ret) return ret; - ret = applesmc_has_key(MOTION_SENSOR_KEY, &s->has_accelerometer); + ret = applesmc_has_key(smc, MOTION_SENSOR_KEY, &s->has_accelerometer); if (ret) return ret; - ret = applesmc_has_key(BACKLIGHT_KEY, &s->has_key_backlight); + ret = applesmc_has_key(smc, BACKLIGHT_KEY, &s->has_key_backlight); if (ret) return ret; @@ -634,13 +896,13 @@ static int applesmc_init_smcreg_try(void) return 0; } -static void applesmc_destroy_smcreg(void) +static void applesmc_destroy_smcreg(struct applesmc_device *smc) { - kfree(smcreg.index); - smcreg.index = NULL; - kfree(smcreg.cache); - smcreg.cache = NULL; - smcreg.init_complete = false; + kfree(smc->reg.index); + smc->reg.index = NULL; + kfree(smc->reg.cache); + smc->reg.cache = NULL; + smc->reg.init_complete = false; } /* @@ -649,12 +911,12 @@ static void applesmc_destroy_smcreg(void) * Retries until initialization is successful, or the operation times out. * */ -static int applesmc_init_smcreg(void) +static int applesmc_init_smcreg(struct applesmc_device *smc) { int ms, ret; for (ms = 0; ms < INIT_TIMEOUT_MSECS; ms += INIT_WAIT_MSECS) { - ret = applesmc_init_smcreg_try(); + ret = applesmc_init_smcreg_try(smc); if (!ret) { if (ms) pr_info("init_smcreg() took %d ms\n", ms); @@ -663,50 +925,223 @@ static int applesmc_init_smcreg(void) msleep(INIT_WAIT_MSECS); } - applesmc_destroy_smcreg(); + applesmc_destroy_smcreg(smc); return ret; } /* Device model stuff */ -static int applesmc_probe(struct platform_device *dev) + +static int applesmc_init_resources(struct applesmc_device *smc); +static void applesmc_free_resources(struct applesmc_device *smc); +static int applesmc_create_modules(struct applesmc_device *smc); +static void applesmc_destroy_modules(struct applesmc_device *smc); + +static int applesmc_add(struct acpi_device *dev) { + struct applesmc_device *smc; int ret; - ret = applesmc_init_smcreg(); + smc = kzalloc(sizeof(struct applesmc_device), GFP_KERNEL); + if (!smc) + return -ENOMEM; + smc->dev = dev; + smc->ldev = &dev->dev; + mutex_init(&smc->reg.mutex); + + dev_set_drvdata(&dev->dev, smc); + + ret = applesmc_init_resources(smc); if (ret) - return ret; + goto out_mem; + + ret = applesmc_init_smcreg(smc); + if (ret) + goto out_res; + + applesmc_device_init(smc); + + ret = applesmc_create_modules(smc); + if (ret) + goto out_reg; + + return 0; + +out_reg: + applesmc_destroy_smcreg(smc); +out_res: + applesmc_free_resources(smc); +out_mem: + dev_set_drvdata(&dev->dev, NULL); + mutex_destroy(&smc->reg.mutex); + kfree(smc); + + return ret; +} + +static void applesmc_remove(struct acpi_device *dev) +{ + struct applesmc_device *smc = dev_get_drvdata(&dev->dev); + + applesmc_destroy_modules(smc); + applesmc_destroy_smcreg(smc); + applesmc_free_resources(smc); - applesmc_device_init(); + mutex_destroy(&smc->reg.mutex); + kfree(smc); + + return; +} + +static acpi_status applesmc_walk_resources(struct acpi_resource *res, + void *data) +{ + struct applesmc_device *smc = data; + + switch (res->type) { + case ACPI_RESOURCE_TYPE_IO: + if (!smc->port_base_set) { + if (res->data.io.address_length < APPLESMC_NR_PORTS) + return AE_ERROR; + smc->port_base = res->data.io.minimum; + smc->port_base_set = true; + } + return AE_OK; + + case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: + if (!smc->iomem_base_set) { + if (res->data.fixed_memory32.address_length < + APPLESMC_IOMEM_MIN_SIZE) { + dev_warn(smc->ldev, "found iomem but it's too small: %u\n", + res->data.fixed_memory32.address_length); + return AE_OK; + } + smc->iomem_base_addr = res->data.fixed_memory32.address; + smc->iomem_base_size = res->data.fixed_memory32.address_length; + smc->iomem_base_set = true; + } + return AE_OK; + + case ACPI_RESOURCE_TYPE_END_TAG: + if (smc->port_base_set) + return AE_OK; + else + return AE_NOT_FOUND; + + default: + return AE_OK; + } +} + +static int applesmc_try_enable_iomem(struct applesmc_device *smc); + +static int applesmc_init_resources(struct applesmc_device *smc) +{ + int ret; + + ret = acpi_walk_resources(smc->dev->handle, METHOD_NAME__CRS, + applesmc_walk_resources, smc); + if (ACPI_FAILURE(ret)) + return -ENXIO; + + if (!request_region(smc->port_base, APPLESMC_NR_PORTS, "applesmc")) + return -ENXIO; + + if (smc->iomem_base_set) { + if (applesmc_try_enable_iomem(smc)) + smc->iomem_base_set = false; + } + + return 0; +} + +static int applesmc_try_enable_iomem(struct applesmc_device *smc) +{ + u8 test_val, ldkn_version; + + dev_dbg(smc->ldev, "Trying to enable iomem based communication\n"); + smc->iomem_base = ioremap(smc->iomem_base_addr, smc->iomem_base_size); + if (!smc->iomem_base) + goto out; + + /* Apple's driver does this check for some reason */ + test_val = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); + if (test_val == 0xff) { + dev_warn(smc->ldev, + "iomem enable failed: initial status is 0xff (is %x)\n", + test_val); + goto out_iomem; + } + + if (read_smc(smc, "LDKN", &ldkn_version, 1)) { + dev_warn(smc->ldev, "iomem enable failed: ldkn read failed\n"); + goto out_iomem; + } + + if (ldkn_version < 2) { + dev_warn(smc->ldev, + "iomem enable failed: ldkn version %u is less than minimum (2)\n", + ldkn_version); + goto out_iomem; + } return 0; + +out_iomem: + iounmap(smc->iomem_base); + +out: + return -ENXIO; +} + +static void applesmc_free_resources(struct applesmc_device *smc) +{ + if (smc->iomem_base_set) + iounmap(smc->iomem_base); + release_region(smc->port_base, APPLESMC_NR_PORTS); } /* Synchronize device with memorized backlight state */ static int applesmc_pm_resume(struct device *dev) { - if (smcreg.has_key_backlight) - applesmc_write_key(BACKLIGHT_KEY, backlight_state, 2); + struct applesmc_device *smc = dev_get_drvdata(dev); + + if (smc->reg.has_key_backlight) + applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); + return 0; } /* Reinitialize device on resume from hibernation */ static int applesmc_pm_restore(struct device *dev) { - applesmc_device_init(); + struct applesmc_device *smc = dev_get_drvdata(dev); + + applesmc_device_init(smc); + return applesmc_pm_resume(dev); } +static const struct acpi_device_id applesmc_ids[] = { + {"APP0001", 0}, + {"", 0}, +}; + static const struct dev_pm_ops applesmc_pm_ops = { .resume = applesmc_pm_resume, .restore = applesmc_pm_restore, }; -static struct platform_driver applesmc_driver = { - .probe = applesmc_probe, - .driver = { - .name = "applesmc", - .pm = &applesmc_pm_ops, +static struct acpi_driver applesmc_driver = { + .name = "applesmc", + .class = "applesmc", + .ids = applesmc_ids, + .ops = { + .add = applesmc_add, + .remove = applesmc_remove + }, + .drv = { + .pm = &applesmc_pm_ops }, }; @@ -714,25 +1149,26 @@ static struct platform_driver applesmc_driver = { * applesmc_calibrate - Set our "resting" values. Callers must * hold applesmc_lock. */ -static void applesmc_calibrate(void) +static void applesmc_calibrate(struct applesmc_device *smc) { - applesmc_read_s16(MOTION_SENSOR_X_KEY, &rest_x); - applesmc_read_s16(MOTION_SENSOR_Y_KEY, &rest_y); - rest_x = -rest_x; + applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &smc->rest_x); + applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &smc->rest_y); + smc->rest_x = -smc->rest_x; } static void applesmc_idev_poll(struct input_dev *idev) { + struct applesmc_device *smc = dev_get_drvdata(&idev->dev); s16 x, y; - if (applesmc_read_s16(MOTION_SENSOR_X_KEY, &x)) + if (applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x)) return; - if (applesmc_read_s16(MOTION_SENSOR_Y_KEY, &y)) + if (applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y)) return; x = -x; - input_report_abs(idev, ABS_X, x - rest_x); - input_report_abs(idev, ABS_Y, y - rest_y); + input_report_abs(idev, ABS_X, x - smc->rest_x); + input_report_abs(idev, ABS_Y, y - smc->rest_y); input_sync(idev); } @@ -747,16 +1183,17 @@ static ssize_t applesmc_name_show(struct device *dev, static ssize_t applesmc_position_show(struct device *dev, struct device_attribute *attr, char *buf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; s16 x, y, z; - ret = applesmc_read_s16(MOTION_SENSOR_X_KEY, &x); + ret = applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x); if (ret) goto out; - ret = applesmc_read_s16(MOTION_SENSOR_Y_KEY, &y); + ret = applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y); if (ret) goto out; - ret = applesmc_read_s16(MOTION_SENSOR_Z_KEY, &z); + ret = applesmc_read_s16(smc, MOTION_SENSOR_Z_KEY, &z); if (ret) goto out; @@ -770,6 +1207,7 @@ static ssize_t applesmc_position_show(struct device *dev, static ssize_t applesmc_light_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; static int data_length; int ret; @@ -777,7 +1215,7 @@ static ssize_t applesmc_light_show(struct device *dev, u8 buffer[10]; if (!data_length) { - entry = applesmc_get_entry_by_key(LIGHT_SENSOR_LEFT_KEY); + entry = applesmc_get_entry_by_key(smc, LIGHT_SENSOR_LEFT_KEY); if (IS_ERR(entry)) return PTR_ERR(entry); if (entry->len > 10) @@ -786,7 +1224,7 @@ static ssize_t applesmc_light_show(struct device *dev, pr_info("light sensor data length set to %d\n", data_length); } - ret = applesmc_read_key(LIGHT_SENSOR_LEFT_KEY, buffer, data_length); + ret = applesmc_read_key(smc, LIGHT_SENSOR_LEFT_KEY, buffer, data_length); if (ret) goto out; /* newer macbooks report a single 10-bit bigendian value */ @@ -796,7 +1234,7 @@ static ssize_t applesmc_light_show(struct device *dev, } left = buffer[2]; - ret = applesmc_read_key(LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); + ret = applesmc_read_key(smc, LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); if (ret) goto out; right = buffer[2]; @@ -812,7 +1250,8 @@ static ssize_t applesmc_light_show(struct device *dev, static ssize_t applesmc_show_sensor_label(struct device *dev, struct device_attribute *devattr, char *sysfsbuf) { - const char *key = smcreg.index[to_index(devattr)]; + struct applesmc_device *smc = dev_get_drvdata(dev); + const char *key = smc->reg.index[to_index(devattr)]; return sysfs_emit(sysfsbuf, "%s\n", key); } @@ -821,12 +1260,13 @@ static ssize_t applesmc_show_sensor_label(struct device *dev, static ssize_t applesmc_show_temperature(struct device *dev, struct device_attribute *devattr, char *sysfsbuf) { - const char *key = smcreg.index[to_index(devattr)]; + struct applesmc_device *smc = dev_get_drvdata(dev); + const char *key = smc->reg.index[to_index(devattr)]; int ret; s16 value; int temp; - ret = applesmc_read_s16(key, &value); + ret = applesmc_read_s16(smc, key, &value); if (ret) return ret; @@ -838,6 +1278,8 @@ static ssize_t applesmc_show_temperature(struct device *dev, static ssize_t applesmc_show_fan_speed(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; int ret; unsigned int speed = 0; char newkey[5]; @@ -846,11 +1288,21 @@ static ssize_t applesmc_show_fan_speed(struct device *dev, scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], to_index(attr)); - ret = applesmc_read_key(newkey, buffer, 2); + entry = applesmc_get_entry_by_key(smc, newkey); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + if (!strcmp(entry->type, FLOAT_TYPE)) { + ret = applesmc_read_entry(smc, entry, (u8 *) &speed, 4); + speed = applesmc_float_to_u32(speed); + } else { + ret = applesmc_read_entry(smc, entry, buffer, 2); + speed = ((buffer[0] << 8 | buffer[1]) >> 2); + } + if (ret) return ret; - speed = ((buffer[0] << 8 | buffer[1]) >> 2); return sysfs_emit(sysfsbuf, "%u\n", speed); } @@ -858,6 +1310,8 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; int ret; unsigned long speed; char newkey[5]; @@ -869,9 +1323,18 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], to_index(attr)); - buffer[0] = (speed >> 6) & 0xff; - buffer[1] = (speed << 2) & 0xff; - ret = applesmc_write_key(newkey, buffer, 2); + entry = applesmc_get_entry_by_key(smc, newkey); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + if (!strcmp(entry->type, FLOAT_TYPE)) { + speed = applesmc_u32_to_float(speed); + ret = applesmc_write_entry(smc, entry, (u8 *) &speed, 4); + } else { + buffer[0] = (speed >> 6) & 0xff; + buffer[1] = (speed << 2) & 0xff; + ret = applesmc_write_key(smc, newkey, buffer, 2); + } if (ret) return ret; @@ -882,15 +1345,30 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, static ssize_t applesmc_show_fan_manual(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; u16 manual = 0; u8 buffer[2]; + char newkey[5]; + bool has_newkey = false; + + scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); + + ret = applesmc_has_key(smc, newkey, &has_newkey); + if (ret) + return ret; + + if (has_newkey) { + ret = applesmc_read_key(smc, newkey, buffer, 1); + manual = buffer[0]; + } else { + ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); + manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; + } - ret = applesmc_read_key(FANS_MANUAL, buffer, 2); if (ret) return ret; - manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; return sysfs_emit(sysfsbuf, "%d\n", manual); } @@ -898,29 +1376,42 @@ static ssize_t applesmc_store_fan_manual(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; u8 buffer[2]; + char newkey[5]; + bool has_newkey = false; unsigned long input; u16 val; if (kstrtoul(sysfsbuf, 10, &input) < 0) return -EINVAL; - ret = applesmc_read_key(FANS_MANUAL, buffer, 2); + scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); + + ret = applesmc_has_key(smc, newkey, &has_newkey); if (ret) - goto out; + return ret; - val = (buffer[0] << 8 | buffer[1]); + if (has_newkey) { + buffer[0] = input & 1; + ret = applesmc_write_key(smc, newkey, buffer, 1); + } else { + ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); + val = (buffer[0] << 8 | buffer[1]); + if (ret) + goto out; - if (input) - val = val | (0x01 << to_index(attr)); - else - val = val & ~(0x01 << to_index(attr)); + if (input) + val = val | (0x01 << to_index(attr)); + else + val = val & ~(0x01 << to_index(attr)); - buffer[0] = (val >> 8) & 0xFF; - buffer[1] = val & 0xFF; + buffer[0] = (val >> 8) & 0xFF; + buffer[1] = val & 0xFF; - ret = applesmc_write_key(FANS_MANUAL, buffer, 2); + ret = applesmc_write_key(smc, FANS_MANUAL, buffer, 2); + } out: if (ret) @@ -932,13 +1423,14 @@ static ssize_t applesmc_store_fan_manual(struct device *dev, static ssize_t applesmc_show_fan_position(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; char newkey[5]; u8 buffer[17]; scnprintf(newkey, sizeof(newkey), FAN_ID_FMT, to_index(attr)); - ret = applesmc_read_key(newkey, buffer, 16); + ret = applesmc_read_key(smc, newkey, buffer, 16); buffer[16] = 0; if (ret) @@ -950,43 +1442,79 @@ static ssize_t applesmc_show_fan_position(struct device *dev, static ssize_t applesmc_calibrate_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { - return sysfs_emit(sysfsbuf, "(%d,%d)\n", rest_x, rest_y); + struct applesmc_device *smc = dev_get_drvdata(dev); + + return sysfs_emit(sysfsbuf, "(%d,%d)\n", smc->rest_x, smc->rest_y); } static ssize_t applesmc_calibrate_store(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { - applesmc_calibrate(); + struct applesmc_device *smc = dev_get_drvdata(dev); + + applesmc_calibrate(smc); return count; } static void applesmc_backlight_set(struct work_struct *work) { - applesmc_write_key(BACKLIGHT_KEY, backlight_state, 2); + struct applesmc_device *smc = container_of(work, struct applesmc_device, backlight_work); + + applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); } -static DECLARE_WORK(backlight_work, &applesmc_backlight_set); static void applesmc_brightness_set(struct led_classdev *led_cdev, enum led_brightness value) { + struct applesmc_device *smc = dev_get_drvdata(led_cdev->dev); int ret; - backlight_state[0] = value; - ret = queue_work(applesmc_led_wq, &backlight_work); + smc->backlight_state[0] = value; + ret = queue_work(smc->backlight_wq, &smc->backlight_work); if (debug && (!ret)) dev_dbg(led_cdev->dev, "work was already on the queue.\n"); } +static ssize_t applesmc_BCLM_store(struct device *dev, + struct device_attribute *attr, char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + u8 val; + + if (kstrtou8(sysfsbuf, 10, &val) < 0) + return -EINVAL; + + if (val < 0 || val > 100) + return -EINVAL; + + if (applesmc_write_key(smc, "BCLM", &val, 1)) + return -ENODEV; + return count; +} + +static ssize_t applesmc_BCLM_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + u8 val; + + if (applesmc_read_key(smc, "BCLM", &val, 1)) + return -ENODEV; + + return sysfs_emit(sysfsbuf, "%d\n", val); +} + static ssize_t applesmc_key_count_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; u8 buffer[4]; u32 count; - ret = applesmc_read_key(KEY_COUNT_KEY, buffer, 4); + ret = applesmc_read_key(smc, KEY_COUNT_KEY, buffer, 4); if (ret) return ret; @@ -998,13 +1526,14 @@ static ssize_t applesmc_key_count_show(struct device *dev, static ssize_t applesmc_key_at_index_read_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; int ret; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); - ret = applesmc_read_entry(entry, sysfsbuf, entry->len); + ret = applesmc_read_entry(smc, entry, sysfsbuf, entry->len); if (ret) return ret; @@ -1014,9 +1543,10 @@ static ssize_t applesmc_key_at_index_read_show(struct device *dev, static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); @@ -1026,9 +1556,10 @@ static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, static ssize_t applesmc_key_at_index_type_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); @@ -1038,9 +1569,10 @@ static ssize_t applesmc_key_at_index_type_show(struct device *dev, static ssize_t applesmc_key_at_index_name_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); @@ -1050,28 +1582,25 @@ static ssize_t applesmc_key_at_index_name_show(struct device *dev, static ssize_t applesmc_key_at_index_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { - return sysfs_emit(sysfsbuf, "%d\n", key_at_index); + struct applesmc_device *smc = dev_get_drvdata(dev); + + return sysfs_emit(sysfsbuf, "%d\n", smc->key_at_index); } static ssize_t applesmc_key_at_index_store(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { + struct applesmc_device *smc = dev_get_drvdata(dev); unsigned long newkey; if (kstrtoul(sysfsbuf, 10, &newkey) < 0 - || newkey >= smcreg.key_count) + || newkey >= smc->reg.key_count) return -EINVAL; - key_at_index = newkey; + smc->key_at_index = newkey; return count; } -static struct led_classdev applesmc_backlight = { - .name = "smc::kbd_backlight", - .default_trigger = "nand-disk", - .brightness_set = applesmc_brightness_set, -}; - static struct applesmc_node_group info_group[] = { { "name", applesmc_name_show }, { "key_count", applesmc_key_count_show }, @@ -1111,19 +1640,25 @@ static struct applesmc_node_group temp_group[] = { { } }; +static struct applesmc_node_group BCLM_group[] = { + { "battery_charge_limit", applesmc_BCLM_show, applesmc_BCLM_store }, + { } +}; + /* Module stuff */ /* * applesmc_destroy_nodes - remove files and free associated memory */ -static void applesmc_destroy_nodes(struct applesmc_node_group *groups) +static void applesmc_destroy_nodes(struct applesmc_device *smc, + struct applesmc_node_group *groups) { struct applesmc_node_group *grp; struct applesmc_dev_attr *node; for (grp = groups; grp->nodes; grp++) { for (node = grp->nodes; node->sda.dev_attr.attr.name; node++) - sysfs_remove_file(&pdev->dev.kobj, + sysfs_remove_file(&smc->dev->dev.kobj, &node->sda.dev_attr.attr); kfree(grp->nodes); grp->nodes = NULL; @@ -1133,7 +1668,8 @@ static void applesmc_destroy_nodes(struct applesmc_node_group *groups) /* * applesmc_create_nodes - create a two-dimensional group of sysfs files */ -static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) +static int applesmc_create_nodes(struct applesmc_device *smc, + struct applesmc_node_group *groups, int num) { struct applesmc_node_group *grp; struct applesmc_dev_attr *node; @@ -1157,7 +1693,7 @@ static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) sysfs_attr_init(attr); attr->name = node->name; attr->mode = 0444 | (grp->store ? 0200 : 0); - ret = sysfs_create_file(&pdev->dev.kobj, attr); + ret = sysfs_create_file(&smc->dev->dev.kobj, attr); if (ret) { attr->name = NULL; goto out; @@ -1167,57 +1703,56 @@ static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) return 0; out: - applesmc_destroy_nodes(groups); + applesmc_destroy_nodes(smc, groups); return ret; } /* Create accelerometer resources */ -static int applesmc_create_accelerometer(void) +static int applesmc_create_accelerometer(struct applesmc_device *smc) { int ret; - - if (!smcreg.has_accelerometer) + if (!smc->reg.has_accelerometer) return 0; - ret = applesmc_create_nodes(accelerometer_group, 1); + ret = applesmc_create_nodes(smc, accelerometer_group, 1); if (ret) goto out; - applesmc_idev = input_allocate_device(); - if (!applesmc_idev) { + smc->idev = input_allocate_device(); + if (!smc->idev) { ret = -ENOMEM; goto out_sysfs; } /* initial calibrate for the input device */ - applesmc_calibrate(); + applesmc_calibrate(smc); /* initialize the input device */ - applesmc_idev->name = "applesmc"; - applesmc_idev->id.bustype = BUS_HOST; - applesmc_idev->dev.parent = &pdev->dev; - input_set_abs_params(applesmc_idev, ABS_X, + smc->idev->name = "applesmc"; + smc->idev->id.bustype = BUS_HOST; + smc->idev->dev.parent = &smc->dev->dev; + input_set_abs_params(smc->idev, ABS_X, -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); - input_set_abs_params(applesmc_idev, ABS_Y, + input_set_abs_params(smc->idev, ABS_Y, -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); - ret = input_setup_polling(applesmc_idev, applesmc_idev_poll); + ret = input_setup_polling(smc->idev, applesmc_idev_poll); if (ret) goto out_idev; - input_set_poll_interval(applesmc_idev, APPLESMC_POLL_INTERVAL); + input_set_poll_interval(smc->idev, APPLESMC_POLL_INTERVAL); - ret = input_register_device(applesmc_idev); + ret = input_register_device(smc->idev); if (ret) goto out_idev; return 0; out_idev: - input_free_device(applesmc_idev); + input_free_device(smc->idev); out_sysfs: - applesmc_destroy_nodes(accelerometer_group); + applesmc_destroy_nodes(smc, accelerometer_group); out: pr_warn("driver init failed (ret=%d)!\n", ret); @@ -1225,44 +1760,55 @@ static int applesmc_create_accelerometer(void) } /* Release all resources used by the accelerometer */ -static void applesmc_release_accelerometer(void) +static void applesmc_release_accelerometer(struct applesmc_device *smc) { - if (!smcreg.has_accelerometer) + if (!smc->reg.has_accelerometer) return; - input_unregister_device(applesmc_idev); - applesmc_destroy_nodes(accelerometer_group); + input_unregister_device(smc->idev); + applesmc_destroy_nodes(smc, accelerometer_group); } -static int applesmc_create_light_sensor(void) +static int applesmc_create_light_sensor(struct applesmc_device *smc) { - if (!smcreg.num_light_sensors) + if (!smc->reg.num_light_sensors) return 0; - return applesmc_create_nodes(light_sensor_group, 1); + return applesmc_create_nodes(smc, light_sensor_group, 1); } -static void applesmc_release_light_sensor(void) +static void applesmc_release_light_sensor(struct applesmc_device *smc) { - if (!smcreg.num_light_sensors) + if (!smc->reg.num_light_sensors) return; - applesmc_destroy_nodes(light_sensor_group); + applesmc_destroy_nodes(smc, light_sensor_group); } -static int applesmc_create_key_backlight(void) +static int applesmc_create_key_backlight(struct applesmc_device *smc) { - if (!smcreg.has_key_backlight) + int ret; + + if (!smc->reg.has_key_backlight) return 0; - applesmc_led_wq = create_singlethread_workqueue("applesmc-led"); - if (!applesmc_led_wq) + smc->backlight_wq = create_singlethread_workqueue("applesmc-led"); + if (!smc->backlight_wq) return -ENOMEM; - return led_classdev_register(&pdev->dev, &applesmc_backlight); + + INIT_WORK(&smc->backlight_work, applesmc_backlight_set); + smc->backlight_dev.name = "smc::kbd_backlight"; + smc->backlight_dev.default_trigger = "nand-disk"; + smc->backlight_dev.brightness_set = applesmc_brightness_set; + ret = led_classdev_register(&smc->dev->dev, &smc->backlight_dev); + if (ret) + destroy_workqueue(smc->backlight_wq); + + return ret; } -static void applesmc_release_key_backlight(void) +static void applesmc_release_key_backlight(struct applesmc_device *smc) { - if (!smcreg.has_key_backlight) + if (!smc->reg.has_key_backlight) return; - led_classdev_unregister(&applesmc_backlight); - destroy_workqueue(applesmc_led_wq); + led_classdev_unregister(&smc->backlight_dev); + destroy_workqueue(smc->backlight_wq); } static int applesmc_dmi_match(const struct dmi_system_id *id) @@ -1291,6 +1837,10 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = { DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), DMI_MATCH(DMI_PRODUCT_NAME, "Macmini") }, }, + { applesmc_dmi_match, "Apple iMacPro", { + DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), + DMI_MATCH(DMI_PRODUCT_NAME, "iMacPro") }, + }, { applesmc_dmi_match, "Apple MacPro", { DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), DMI_MATCH(DMI_PRODUCT_NAME, "MacPro") }, @@ -1306,90 +1856,91 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = { { .ident = NULL } }; -static int __init applesmc_init(void) +static int applesmc_create_modules(struct applesmc_device *smc) { int ret; - if (!dmi_check_system(applesmc_whitelist)) { - pr_warn("supported laptop not found!\n"); - ret = -ENODEV; - goto out; - } - - if (!request_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS, - "applesmc")) { - ret = -ENXIO; - goto out; - } - - ret = platform_driver_register(&applesmc_driver); - if (ret) - goto out_region; - - pdev = platform_device_register_simple("applesmc", APPLESMC_DATA_PORT, - NULL, 0); - if (IS_ERR(pdev)) { - ret = PTR_ERR(pdev); - goto out_driver; - } - - /* create register cache */ - ret = applesmc_init_smcreg(); + ret = applesmc_create_nodes(smc, info_group, 1); if (ret) - goto out_device; - - ret = applesmc_create_nodes(info_group, 1); + goto out; + ret = applesmc_create_nodes(smc, BCLM_group, 1); if (ret) - goto out_smcreg; + goto out_info; - ret = applesmc_create_nodes(fan_group, smcreg.fan_count); + ret = applesmc_create_nodes(smc, fan_group, smc->reg.fan_count); if (ret) - goto out_info; + goto out_bclm; - ret = applesmc_create_nodes(temp_group, smcreg.index_count); + ret = applesmc_create_nodes(smc, temp_group, smc->reg.index_count); if (ret) goto out_fans; - ret = applesmc_create_accelerometer(); + ret = applesmc_create_accelerometer(smc); if (ret) goto out_temperature; - ret = applesmc_create_light_sensor(); + ret = applesmc_create_light_sensor(smc); if (ret) goto out_accelerometer; - ret = applesmc_create_key_backlight(); + ret = applesmc_create_key_backlight(smc); if (ret) goto out_light_sysfs; - hwmon_dev = hwmon_device_register(&pdev->dev); - if (IS_ERR(hwmon_dev)) { - ret = PTR_ERR(hwmon_dev); + smc->hwmon_dev = hwmon_device_register(&smc->dev->dev); + if (IS_ERR(smc->hwmon_dev)) { + ret = PTR_ERR(smc->hwmon_dev); goto out_light_ledclass; } return 0; out_light_ledclass: - applesmc_release_key_backlight(); + applesmc_release_key_backlight(smc); out_light_sysfs: - applesmc_release_light_sensor(); + applesmc_release_light_sensor(smc); out_accelerometer: - applesmc_release_accelerometer(); + applesmc_release_accelerometer(smc); out_temperature: - applesmc_destroy_nodes(temp_group); + applesmc_destroy_nodes(smc, temp_group); out_fans: - applesmc_destroy_nodes(fan_group); + applesmc_destroy_nodes(smc, fan_group); +out_bclm: + applesmc_destroy_nodes(smc, BCLM_group); out_info: - applesmc_destroy_nodes(info_group); -out_smcreg: - applesmc_destroy_smcreg(); -out_device: - platform_device_unregister(pdev); -out_driver: - platform_driver_unregister(&applesmc_driver); -out_region: - release_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS); + applesmc_destroy_nodes(smc, info_group); +out: + return ret; +} + +static void applesmc_destroy_modules(struct applesmc_device *smc) +{ + hwmon_device_unregister(smc->hwmon_dev); + applesmc_release_key_backlight(smc); + applesmc_release_light_sensor(smc); + applesmc_release_accelerometer(smc); + applesmc_destroy_nodes(smc, temp_group); + applesmc_destroy_nodes(smc, fan_group); + applesmc_destroy_nodes(smc, BCLM_group); + applesmc_destroy_nodes(smc, info_group); +} + +static int __init applesmc_init(void) +{ + int ret; + + if (!dmi_check_system(applesmc_whitelist)) { + pr_warn("supported laptop not found!\n"); + ret = -ENODEV; + goto out; + } + + ret = acpi_bus_register_driver(&applesmc_driver); + if (ret) + goto out; + + return 0; + out: pr_warn("driver init failed (ret=%d)!\n", ret); return ret; @@ -1397,23 +1948,14 @@ static int __init applesmc_init(void) static void __exit applesmc_exit(void) { - hwmon_device_unregister(hwmon_dev); - applesmc_release_key_backlight(); - applesmc_release_light_sensor(); - applesmc_release_accelerometer(); - applesmc_destroy_nodes(temp_group); - applesmc_destroy_nodes(fan_group); - applesmc_destroy_nodes(info_group); - applesmc_destroy_smcreg(); - platform_device_unregister(pdev); - platform_driver_unregister(&applesmc_driver); - release_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS); + acpi_bus_unregister_driver(&applesmc_driver); } module_init(applesmc_init); module_exit(applesmc_exit); MODULE_AUTHOR("Nicolas Boichat"); +MODULE_AUTHOR("Paul Pawlowski"); MODULE_DESCRIPTION("Apple SMC"); MODULE_LICENSE("GPL v2"); MODULE_DEVICE_TABLE(dmi, applesmc_whitelist); diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c index 78748e8d2dba..2b2b558cebe6 100644 --- a/drivers/pci/vgaarb.c +++ b/drivers/pci/vgaarb.c @@ -143,6 +143,7 @@ void vga_set_default_device(struct pci_dev *pdev) pci_dev_put(vga_default); vga_default = pci_dev_get(pdev); } +EXPORT_SYMBOL_GPL(vga_set_default_device); /** * vga_remove_vgacon - deactivate VGA console diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c index 1417e230edbd..e69785af8e1d 100644 --- a/drivers/platform/x86/apple-gmux.c +++ b/drivers/platform/x86/apple-gmux.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,10 @@ struct apple_gmux_config { # define MMIO_GMUX_MAX_BRIGHTNESS 0xffff +static bool force_igd; +module_param(force_igd, bool, 0); +MODULE_PARM_DESC(force_idg, "Switch gpu to igd on module load. Make sure that you have apple-set-os set up and the iGPU is in `lspci -s 00:02.0`. (default: false) (bool)"); + static u8 gmux_pio_read8(struct apple_gmux_data *gmux_data, int port) { return inb(gmux_data->iostart + port); @@ -945,6 +950,19 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) gmux_enable_interrupts(gmux_data); gmux_read_switch_state(gmux_data); + if (force_igd) { + struct pci_dev *pdev; + + pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(2, 0)); + if (pdev) { + pr_info("Switching to IGD"); + gmux_switchto(VGA_SWITCHEROO_IGD); + vga_set_default_device(pdev); + } else { + pr_err("force_idg is true, but couldn't find iGPU at 00:02.0! Is apple-set-os working?"); + } + } + /* * Retina MacBook Pros cannot switch the panel's AUX separately * and need eDP pre-calibration. They are distinguishable from diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 6388cbe1e56b..50f092732796 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,16 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + config APPLE_MAILBOX tristate "Apple SoC mailboxes" depends on PM @@ -30,6 +40,20 @@ config APPLE_RTKIT Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 4d9ab8f3037b..5e526a9edcf2 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,10 +1,16 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c new file mode 100644 index 000000000000..3a0d7964007c --- /dev/null +++ b/drivers/soc/apple/dockchannel.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + struct device_node *child; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq <= 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + for_each_child_of_node(dev->of_node, child) + of_platform_device_create(child, NULL, dev); + + return 0; +} + +static void dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c new file mode 100644 index 000000000000..080d083ed9bd --- /dev/null +++ b/drivers/soc/apple/rtkit-helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static void apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index 075e775d3868..e1cc0d60eeb6 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -50,4 +50,6 @@ source "drivers/staging/vme_user/Kconfig" source "drivers/staging/gpib/Kconfig" +source "drivers/staging/apple-bce/Kconfig" + endif # STAGING diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index e681e403509c..4045c588b3b4 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -14,3 +14,4 @@ obj-$(CONFIG_GREYBUS) += greybus/ obj-$(CONFIG_BCM2835_VCHIQ) += vc04_services/ obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ obj-$(CONFIG_GPIB) += gpib/ +obj-$(CONFIG_APPLE_BCE) += apple-bce/ diff --git a/drivers/staging/apple-bce/Kconfig b/drivers/staging/apple-bce/Kconfig new file mode 100644 index 000000000000..fe92bc441e89 --- /dev/null +++ b/drivers/staging/apple-bce/Kconfig @@ -0,0 +1,18 @@ +config APPLE_BCE + tristate "Apple BCE driver (VHCI and Audio support)" + default m + depends on X86 + select SOUND + select SND + select SND_PCM + select SND_JACK + help + VHCI and audio support on Apple MacBooks with the T2 Chip. + This driver is divided in three components: + - BCE (Buffer Copy Engine): which establishes a basic communication + channel with the T2 chip. This component is required by the other two: + - VHCI (Virtual Host Controller Interface): Access to keyboard, mouse + and other system devices depend on this virtual USB host controller + - Audio: a driver for the T2 audio interface. + + If "M" is selected, the module will be called apple-bce.' diff --git a/drivers/staging/apple-bce/Makefile b/drivers/staging/apple-bce/Makefile new file mode 100644 index 000000000000..8cfbd3f64af6 --- /dev/null +++ b/drivers/staging/apple-bce/Makefile @@ -0,0 +1,28 @@ +modname := apple-bce +obj-$(CONFIG_APPLE_BCE) += $(modname).o + +apple-bce-objs := apple_bce.o mailbox.o queue.o queue_dma.o vhci/vhci.o vhci/queue.o vhci/transfer.o audio/audio.o audio/protocol.o audio/protocol_bce.o audio/pcm.o + +MY_CFLAGS += -DWITHOUT_NVME_PATCH +#MY_CFLAGS += -g -DDEBUG +ccflags-y += ${MY_CFLAGS} +CC += ${MY_CFLAGS} + +KVERSION := $(KERNELRELEASE) +ifeq ($(origin KERNELRELEASE), undefined) +KVERSION := $(shell uname -r) +endif + +KDIR := /lib/modules/$(KVERSION)/build +PWD := $(shell pwd) + +.PHONY: all + +all: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean + +install: + $(MAKE) -C $(KDIR) M=$(PWD) modules_install diff --git a/drivers/staging/apple-bce/apple_bce.c b/drivers/staging/apple-bce/apple_bce.c new file mode 100644 index 000000000000..4fd2415d7028 --- /dev/null +++ b/drivers/staging/apple-bce/apple_bce.c @@ -0,0 +1,445 @@ +#include "apple_bce.h" +#include +#include +#include "audio/audio.h" +#include + +static dev_t bce_chrdev; +static struct class *bce_class; + +struct apple_bce_device *global_bce; + +static int bce_create_command_queues(struct apple_bce_device *bce); +static void bce_free_command_queues(struct apple_bce_device *bce); +static irqreturn_t bce_handle_mb_irq(int irq, void *dev); +static irqreturn_t bce_handle_dma_irq(int irq, void *dev); +static int bce_fw_version_handshake(struct apple_bce_device *bce); +static int bce_register_command_queue(struct apple_bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq); + +static int apple_bce_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct apple_bce_device *bce = NULL; + int status = 0; + int nvec; + + pr_info("apple-bce: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "apple-bce")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + nvec = pci_alloc_irq_vectors(dev, 1, 8, PCI_IRQ_MSI); + if (nvec < 5) { + status = -EINVAL; + goto fail; + } + + bce = kzalloc(sizeof(struct apple_bce_device), GFP_KERNEL); + if (!bce) { + status = -ENOMEM; + goto fail; + } + + bce->pci = dev; + pci_set_drvdata(dev, bce); + + bce->devt = bce_chrdev; + bce->dev = device_create(bce_class, &dev->dev, bce->devt, NULL, "apple-bce"); + if (IS_ERR_OR_NULL(bce->dev)) { + status = PTR_ERR(bce_class); + goto fail; + } + + bce->reg_mem_mb = pci_iomap(dev, 4, 0); + bce->reg_mem_dma = pci_iomap(dev, 2, 0); + + if (IS_ERR_OR_NULL(bce->reg_mem_mb) || IS_ERR_OR_NULL(bce->reg_mem_dma)) { + dev_warn(&dev->dev, "apple-bce: Failed to pci_iomap required regions\n"); + goto fail; + } + + bce_mailbox_init(&bce->mbox, bce->reg_mem_mb); + bce_timestamp_init(&bce->timestamp, bce->reg_mem_mb); + + spin_lock_init(&bce->queues_lock); + ida_init(&bce->queue_ida); + + if ((status = pci_request_irq(dev, 0, bce_handle_mb_irq, NULL, dev, "bce_mbox"))) + goto fail; + if ((status = pci_request_irq(dev, 4, NULL, bce_handle_dma_irq, dev, "bce_dma"))) + goto fail_interrupt_0; + + if ((status = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(37)))) { + dev_warn(&dev->dev, "dma: Setting mask failed\n"); + goto fail_interrupt; + } + + /* Gets the function 0's interface. This is needed because Apple only accepts DMA on our function if function 0 + is a bus master, so we need to work around this. */ + bce->pci0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); +#ifndef WITHOUT_NVME_PATCH + if ((status = pci_enable_device_mem(bce->pci0))) { + dev_warn(&dev->dev, "apple-bce: failed to enable function 0\n"); + goto fail_dev0; + } +#endif + pci_set_master(bce->pci0); + + bce_timestamp_start(&bce->timestamp, true); + + if ((status = bce_fw_version_handshake(bce))) + goto fail_ts; + pr_info("apple-bce: handshake done\n"); + + if ((status = bce_create_command_queues(bce))) { + pr_info("apple-bce: Creating command queues failed\n"); + goto fail_ts; + } + + global_bce = bce; + + bce_vhci_create(bce, &bce->vhci); + + return 0; + +fail_ts: + bce_timestamp_stop(&bce->timestamp); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +fail_dev0: +#endif + pci_dev_put(bce->pci0); +fail_interrupt: + pci_free_irq(dev, 4, dev); +fail_interrupt_0: + pci_free_irq(dev, 0, dev); +fail: + if (bce && bce->dev) { + device_destroy(bce_class, bce->devt); + + if (!IS_ERR_OR_NULL(bce->reg_mem_mb)) + pci_iounmap(dev, bce->reg_mem_mb); + if (!IS_ERR_OR_NULL(bce->reg_mem_dma)) + pci_iounmap(dev, bce->reg_mem_dma); + + kfree(bce); + } + + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + +static int bce_create_command_queues(struct apple_bce_device *bce) +{ + int status; + struct bce_queue_memcfg *cfg; + + bce->cmd_cq = bce_alloc_cq(bce, 0, 0x20); + bce->cmd_cmdq = bce_alloc_cmdq(bce, 1, 0x20); + if (bce->cmd_cq == NULL || bce->cmd_cmdq == NULL) { + status = -ENOMEM; + goto err; + } + bce->queues[0] = (struct bce_queue *) bce->cmd_cq; + bce->queues[1] = (struct bce_queue *) bce->cmd_cmdq->sq; + + cfg = kzalloc(sizeof(struct bce_queue_memcfg), GFP_KERNEL); + if (!cfg) { + status = -ENOMEM; + goto err; + } + bce_get_cq_memcfg(bce->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, false))) + goto err; + bce_get_sq_memcfg(bce->cmd_cmdq->sq, bce->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, true))) + goto err; + kfree(cfg); + + return 0; + +err: + if (bce->cmd_cq) + bce_free_cq(bce, bce->cmd_cq); + if (bce->cmd_cmdq) + bce_free_cmdq(bce, bce->cmd_cmdq); + return status; +} + +static void bce_free_command_queues(struct apple_bce_device *bce) +{ + bce_free_cq(bce, bce->cmd_cq); + bce_free_cmdq(bce, bce->cmd_cmdq); + bce->cmd_cq = NULL; + bce->queues[0] = NULL; +} + +static irqreturn_t bce_handle_mb_irq(int irq, void *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(dev); + bce_mailbox_handle_interrupt(&bce->mbox); + return IRQ_HANDLED; +} + +static irqreturn_t bce_handle_dma_irq(int irq, void *dev) +{ + int i; + struct apple_bce_device *bce = pci_get_drvdata(dev); + spin_lock(&bce->queues_lock); + for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) + if (bce->queues[i] && bce->queues[i]->type == BCE_QUEUE_CQ) + bce_handle_cq_completions(bce, (struct bce_queue_cq *) bce->queues[i]); + spin_unlock(&bce->queues_lock); + return IRQ_HANDLED; +} + +static int bce_fw_version_handshake(struct apple_bce_device *bce) +{ + u64 result; + int status; + + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VERSION, BC_PROTOCOL_VERSION), + &result))) + return status; + if (BCE_MB_TYPE(result) != BCE_MB_SET_FW_PROTOCOL_VERSION || + BCE_MB_VALUE(result) != BC_PROTOCOL_VERSION) { + pr_err("apple-bce: FW version handshake failed %x:%llx\n", BCE_MB_TYPE(result), BCE_MB_VALUE(result)); + return -EINVAL; + } + return 0; +} + +static int bce_register_command_queue(struct apple_bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq) +{ + int status; + int cmd_type; + u64 result; + // OS X uses an bidirectional direction, but that's not really needed + dma_addr_t a = dma_map_single(&bce->pci->dev, cfg, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (dma_mapping_error(&bce->pci->dev, a)) + return -ENOMEM; + cmd_type = is_sq ? BCE_MB_REGISTER_COMMAND_SQ : BCE_MB_REGISTER_COMMAND_CQ; + status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(cmd_type, a), &result); + dma_unmap_single(&bce->pci->dev, a, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (status) + return status; + if (BCE_MB_TYPE(result) != BCE_MB_REGISTER_COMMAND_QUEUE_REPLY) + return -EINVAL; + return 0; +} + +static void apple_bce_remove(struct pci_dev *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(dev); + bce->is_being_removed = true; + + bce_vhci_destroy(&bce->vhci); + + bce_timestamp_stop(&bce->timestamp); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +#endif + pci_dev_put(bce->pci0); + pci_free_irq(dev, 0, dev); + pci_free_irq(dev, 4, dev); + bce_free_command_queues(bce); + pci_iounmap(dev, bce->reg_mem_mb); + pci_iounmap(dev, bce->reg_mem_dma); + device_destroy(bce_class, bce->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + kfree(bce); +} + +static int bce_save_state_and_sleep(struct apple_bce_device *bce) +{ + int attempt, status = 0; + u64 resp; + dma_addr_t dma_addr; + void *dma_ptr = NULL; + size_t size = max(PAGE_SIZE, 4096UL); + + for (attempt = 0; attempt < 5; ++attempt) { + pr_debug("apple-bce: suspend: attempt %i, buffer size %li\n", attempt, size); + dma_ptr = dma_alloc_coherent(&bce->pci->dev, size, &dma_addr, GFP_KERNEL); + if (!dma_ptr) { + pr_err("apple-bce: suspend failed (data alloc failed)\n"); + break; + } + BUG_ON((dma_addr % 4096) != 0); + status = bce_mailbox_send(&bce->mbox, + BCE_MB_MSG(BCE_MB_SAVE_STATE_AND_SLEEP, (dma_addr & ~(4096LLU - 1)) | (size / 4096)), &resp); + if (status) { + pr_err("apple-bce: suspend failed (mailbox send)\n"); + break; + } + if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + bce->saved_data_dma_addr = dma_addr; + bce->saved_data_dma_ptr = dma_ptr; + bce->saved_data_dma_size = size; + return 0; + } else if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_STATE_AND_SLEEP_FAILURE) { + dma_free_coherent(&bce->pci->dev, size, dma_ptr, dma_addr); + /* The 0x10ff magic value was extracted from Apple's driver */ + size = (BCE_MB_VALUE(resp) + 0x10ff) & ~(4096LLU - 1); + pr_debug("apple-bce: suspend: device requested a larger buffer (%li)\n", size); + continue; + } else { + pr_err("apple-bce: suspend failed (invalid device response)\n"); + status = -EINVAL; + break; + } + } + if (dma_ptr) + dma_free_coherent(&bce->pci->dev, size, dma_ptr, dma_addr); + if (!status) + return bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), &resp); + return status; +} + +static int bce_restore_state_and_wake(struct apple_bce_device *bce) +{ + int status; + u64 resp; + if (!bce->saved_data_dma_ptr) { + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &resp))) { + pr_err("apple-bce: resume with no state failed (mailbox send)\n"); + return status; + } + if (BCE_MB_TYPE(resp) != BCE_MB_RESTORE_NO_STATE) { + pr_err("apple-bce: resume with no state failed (invalid device response)\n"); + return -EINVAL; + } + return 0; + } + + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_STATE_AND_WAKE, + (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), &resp))) { + pr_err("apple-bce: resume with state failed (mailbox send)\n"); + goto finish_with_state; + } + if (BCE_MB_TYPE(resp) != BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + pr_err("apple-bce: resume with state failed (invalid device response)\n"); + status = -EINVAL; + goto finish_with_state; + } + +finish_with_state: + dma_free_coherent(&bce->pci->dev, bce->saved_data_dma_size, bce->saved_data_dma_ptr, bce->saved_data_dma_addr); + bce->saved_data_dma_ptr = NULL; + return status; +} + +static int apple_bce_suspend(struct device *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + + bce_timestamp_stop(&bce->timestamp); + + if ((status = bce_save_state_and_sleep(bce))) + return status; + + return 0; +} + +static int apple_bce_resume(struct device *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + + pci_set_master(bce->pci); + pci_set_master(bce->pci0); + + if ((status = bce_restore_state_and_wake(bce))) + return status; + + bce_timestamp_start(&bce->timestamp, false); + + return 0; +} + +static struct pci_device_id apple_bce_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1801) }, + { 0, }, +}; + +MODULE_DEVICE_TABLE(pci, apple_bce_ids); + +struct dev_pm_ops apple_bce_pci_driver_pm = { + .suspend = apple_bce_suspend, + .resume = apple_bce_resume +}; +struct pci_driver apple_bce_pci_driver = { + .name = "apple-bce", + .id_table = apple_bce_ids, + .probe = apple_bce_probe, + .remove = apple_bce_remove, + .driver = { + .pm = &apple_bce_pci_driver_pm + } +}; + + +static int __init apple_bce_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&bce_chrdev, 0, 1, "apple-bce"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_class = class_create(THIS_MODULE, "apple-bce"); +#else + bce_class = class_create("apple-bce"); +#endif + if (IS_ERR(bce_class)) { + result = PTR_ERR(bce_class); + goto fail_class; + } + if ((result = bce_vhci_module_init())) { + pr_err("apple-bce: bce-vhci init failed"); + goto fail_class; + } + + result = pci_register_driver(&apple_bce_pci_driver); + if (result) + goto fail_drv; + + aaudio_module_init(); + + return 0; + +fail_drv: + pci_unregister_driver(&apple_bce_pci_driver); +fail_class: + class_destroy(bce_class); +fail_chrdev: + unregister_chrdev_region(bce_chrdev, 1); + if (!result) + result = -EINVAL; + return result; +} +static void __exit apple_bce_module_exit(void) +{ + pci_unregister_driver(&apple_bce_pci_driver); + + aaudio_module_exit(); + bce_vhci_module_exit(); + class_destroy(bce_class); + unregister_chrdev_region(bce_chrdev, 1); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("MrARM"); +MODULE_DESCRIPTION("Apple BCE Driver"); +MODULE_VERSION("0.01"); +module_init(apple_bce_module_init); +module_exit(apple_bce_module_exit); diff --git a/drivers/staging/apple-bce/apple_bce.h b/drivers/staging/apple-bce/apple_bce.h new file mode 100644 index 000000000000..58dbeff79e43 --- /dev/null +++ b/drivers/staging/apple-bce/apple_bce.h @@ -0,0 +1,41 @@ +#ifndef APPLE_BCE_H +#define APPLE_BCE_H + +#include +#include +#include "mailbox.h" +#include "queue.h" +#include "vhci/vhci.h" + +#define BC_PROTOCOL_VERSION 0x20001 +#define BCE_MAX_QUEUE_COUNT 0x100 + +#define BCE_QUEUE_USER_MIN 2 +#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) + +struct apple_bce_device { + struct pci_dev *pci, *pci0; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_mb; + void __iomem *reg_mem_dma; + struct bce_mailbox mbox; + struct bce_timestamp timestamp; + struct bce_queue *queues[BCE_MAX_QUEUE_COUNT]; + struct spinlock queues_lock; + struct ida queue_ida; + struct bce_queue_cq *cmd_cq; + struct bce_queue_cmdq *cmd_cmdq; + struct bce_queue_sq *int_sq_list[BCE_MAX_QUEUE_COUNT]; + bool is_being_removed; + + dma_addr_t saved_data_dma_addr; + void *saved_data_dma_ptr; + size_t saved_data_dma_size; + + struct bce_vhci vhci; +}; + +extern struct apple_bce_device *global_bce; + +#endif //APPLE_BCE_H diff --git a/drivers/staging/apple-bce/audio/audio.c b/drivers/staging/apple-bce/audio/audio.c new file mode 100644 index 000000000000..bd16ddd16c1d --- /dev/null +++ b/drivers/staging/apple-bce/audio/audio.c @@ -0,0 +1,711 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "audio.h" +#include "pcm.h" +#include + +static int aaudio_alsa_index = SNDRV_DEFAULT_IDX1; +static char *aaudio_alsa_id = SNDRV_DEFAULT_STR1; + +static dev_t aaudio_chrdev; +static struct class *aaudio_class; + +static int aaudio_init_cmd(struct aaudio_device *a); +static int aaudio_init_bs(struct aaudio_device *a); +static void aaudio_init_dev(struct aaudio_device *a, aaudio_device_id_t dev_id); +static void aaudio_free_dev(struct aaudio_subdevice *sdev); + +static int aaudio_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct aaudio_device *aaudio = NULL; + struct aaudio_subdevice *sdev = NULL; + int status = 0; + u32 cfg; + + pr_info("aaudio: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "aaudio")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + + aaudio = kzalloc(sizeof(struct aaudio_device), GFP_KERNEL); + if (!aaudio) { + status = -ENOMEM; + goto fail; + } + + aaudio->bce = global_bce; + if (!aaudio->bce) { + dev_warn(&dev->dev, "aaudio: No BCE available\n"); + status = -EINVAL; + goto fail; + } + + aaudio->pci = dev; + pci_set_drvdata(dev, aaudio); + + aaudio->devt = aaudio_chrdev; + aaudio->dev = device_create(aaudio_class, &dev->dev, aaudio->devt, NULL, "aaudio"); + if (IS_ERR_OR_NULL(aaudio->dev)) { + status = PTR_ERR(aaudio_class); + goto fail; + } + device_link_add(aaudio->dev, aaudio->bce->dev, DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); + + init_completion(&aaudio->remote_alive); + INIT_LIST_HEAD(&aaudio->subdevice_list); + + /* Init: set an unknown flag in the bitset */ + if (pci_read_config_dword(dev, 4, &cfg)) + dev_warn(&dev->dev, "aaudio: pci_read_config_dword fail\n"); + if (pci_write_config_dword(dev, 4, cfg | 6u)) + dev_warn(&dev->dev, "aaudio: pci_write_config_dword fail\n"); + + dev_info(aaudio->dev, "aaudio: bs len = %llx\n", pci_resource_len(dev, 0)); + aaudio->reg_mem_bs_dma = pci_resource_start(dev, 0); + aaudio->reg_mem_bs = pci_iomap(dev, 0, 0); + aaudio->reg_mem_cfg = pci_iomap(dev, 4, 0); + + aaudio->reg_mem_gpr = (u32 __iomem *) ((u8 __iomem *) aaudio->reg_mem_cfg + 0xC000); + + if (IS_ERR_OR_NULL(aaudio->reg_mem_bs) || IS_ERR_OR_NULL(aaudio->reg_mem_cfg)) { + dev_warn(&dev->dev, "aaudio: Failed to pci_iomap required regions\n"); + goto fail; + } + + if (aaudio_bce_init(aaudio)) { + dev_warn(&dev->dev, "aaudio: Failed to init BCE command transport\n"); + goto fail; + } + + if (snd_card_new(aaudio->dev, aaudio_alsa_index, aaudio_alsa_id, THIS_MODULE, 0, &aaudio->card)) { + dev_err(&dev->dev, "aaudio: Failed to create ALSA card\n"); + goto fail; + } + + strcpy(aaudio->card->shortname, "Apple T2 Audio"); + strcpy(aaudio->card->longname, "Apple T2 Audio"); + strcpy(aaudio->card->mixername, "Apple T2 Audio"); + /* Dynamic alsa ids start at 100 */ + aaudio->next_alsa_id = 100; + + if (aaudio_init_cmd(aaudio)) { + dev_err(&dev->dev, "aaudio: Failed to initialize over BCE\n"); + goto fail_snd; + } + + if (aaudio_init_bs(aaudio)) { + dev_err(&dev->dev, "aaudio: Failed to initialize BufferStruct\n"); + goto fail_snd; + } + + if ((status = aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_ON))) { + dev_err(&dev->dev, "Failed to set remote access\n"); + return status; + } + + if (snd_card_register(aaudio->card)) { + dev_err(&dev->dev, "aaudio: Failed to register ALSA sound device\n"); + goto fail_snd; + } + + list_for_each_entry(sdev, &aaudio->subdevice_list, list) { + struct aaudio_buffer_struct_device *dev = &aaudio->bs->devices[sdev->buf_id]; + + if (sdev->out_stream_cnt == 1 && !strcmp(dev->name, "Speaker")) { + struct snd_pcm_hardware *hw = sdev->out_streams[0].alsa_hw_desc; + + snprintf(aaudio->card->driver, sizeof(aaudio->card->driver) / sizeof(char), "AppleT2x%d", hw->channels_min); + } + } + + return 0; + +fail_snd: + snd_card_free(aaudio->card); +fail: + if (aaudio && aaudio->dev) + device_destroy(aaudio_class, aaudio->devt); + kfree(aaudio); + + if (!IS_ERR_OR_NULL(aaudio->reg_mem_bs)) + pci_iounmap(dev, aaudio->reg_mem_bs); + if (!IS_ERR_OR_NULL(aaudio->reg_mem_cfg)) + pci_iounmap(dev, aaudio->reg_mem_cfg); + + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + + + +static void aaudio_remove(struct pci_dev *dev) +{ + struct aaudio_subdevice *sdev; + struct aaudio_device *aaudio = pci_get_drvdata(dev); + + snd_card_free(aaudio->card); + while (!list_empty(&aaudio->subdevice_list)) { + sdev = list_first_entry(&aaudio->subdevice_list, struct aaudio_subdevice, list); + list_del(&sdev->list); + aaudio_free_dev(sdev); + } + pci_iounmap(dev, aaudio->reg_mem_bs); + pci_iounmap(dev, aaudio->reg_mem_cfg); + device_destroy(aaudio_class, aaudio->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + kfree(aaudio); +} + +static int aaudio_suspend(struct device *dev) +{ + struct aaudio_device *aaudio = pci_get_drvdata(to_pci_dev(dev)); + + if (aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_OFF)) + dev_warn(aaudio->dev, "Failed to reset remote access\n"); + + pci_disable_device(aaudio->pci); + return 0; +} + +static int aaudio_resume(struct device *dev) +{ + int status; + struct aaudio_device *aaudio = pci_get_drvdata(to_pci_dev(dev)); + + if ((status = pci_enable_device(aaudio->pci))) + return status; + pci_set_master(aaudio->pci); + + if ((status = aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_ON))) { + dev_err(aaudio->dev, "Failed to set remote access\n"); + return status; + } + + return 0; +} + +static int aaudio_init_cmd(struct aaudio_device *a) +{ + int status; + struct aaudio_send_ctx sctx; + struct aaudio_msg buf; + u64 dev_cnt, dev_i; + aaudio_device_id_t *dev_l; + + if ((status = aaudio_send(a, &sctx, 500, + aaudio_msg_write_alive_notification, 1, 3))) { + dev_err(a->dev, "Sending alive notification failed\n"); + return status; + } + + if (wait_for_completion_timeout(&a->remote_alive, msecs_to_jiffies(500)) == 0) { + dev_err(a->dev, "Timed out waiting for remote\n"); + return -ETIMEDOUT; + } + dev_info(a->dev, "Continuing init\n"); + + buf = aaudio_reply_alloc(); + if ((status = aaudio_cmd_get_device_list(a, &buf, &dev_l, &dev_cnt))) { + dev_err(a->dev, "Failed to get device list\n"); + aaudio_reply_free(&buf); + return status; + } + for (dev_i = 0; dev_i < dev_cnt; ++dev_i) + aaudio_init_dev(a, dev_l[dev_i]); + aaudio_reply_free(&buf); + + return 0; +} + +static void aaudio_init_stream_info(struct aaudio_subdevice *sdev, struct aaudio_stream *strm); +static void aaudio_handle_jack_connection_change(struct aaudio_subdevice *sdev); + +static void aaudio_init_dev(struct aaudio_device *a, aaudio_device_id_t dev_id) +{ + struct aaudio_subdevice *sdev; + struct aaudio_msg buf = aaudio_reply_alloc(); + u64 uid_len, stream_cnt, i; + aaudio_object_id_t *stream_list; + char *uid; + + sdev = kzalloc(sizeof(struct aaudio_subdevice), GFP_KERNEL); + + if (aaudio_cmd_get_property(a, &buf, dev_id, dev_id, AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_UID, 0), + NULL, 0, (void **) &uid, &uid_len) || uid_len > AAUDIO_DEVICE_MAX_UID_LEN) { + dev_err(a->dev, "Failed to get device uid for device %llx\n", dev_id); + goto fail; + } + dev_info(a->dev, "Remote device %llx %.*s\n", dev_id, (int) uid_len, uid); + + sdev->a = a; + INIT_LIST_HEAD(&sdev->list); + sdev->dev_id = dev_id; + sdev->buf_id = AAUDIO_BUFFER_ID_NONE; + strncpy(sdev->uid, uid, uid_len); + sdev->uid[uid_len + 1] = '\0'; + + if (aaudio_cmd_get_primitive_property(a, dev_id, dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_INPUT, AAUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->in_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device input latency\n"); + if (aaudio_cmd_get_primitive_property(a, dev_id, dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->out_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device output latency\n"); + + if (aaudio_cmd_get_input_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get input stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > AAUDIO_DEIVCE_MAX_INPUT_STREAMS) { + dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, AAUDIO_DEIVCE_MAX_INPUT_STREAMS); + stream_cnt = AAUDIO_DEIVCE_MAX_INPUT_STREAMS; + } + sdev->in_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->in_streams[i].id = stream_list[i]; + sdev->in_streams[i].buffer_cnt = 0; + aaudio_init_stream_info(sdev, &sdev->in_streams[i]); + sdev->in_streams[i].latency += sdev->in_latency; + } + + if (aaudio_cmd_get_output_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get output stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS) { + dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS); + stream_cnt = AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS; + } + sdev->out_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->out_streams[i].id = stream_list[i]; + sdev->out_streams[i].buffer_cnt = 0; + aaudio_init_stream_info(sdev, &sdev->out_streams[i]); + sdev->out_streams[i].latency += sdev->in_latency; + } + + if (sdev->is_pcm) + aaudio_create_pcm(sdev); + /* Headphone Jack status */ + if (!strcmp(sdev->uid, "Codec Output")) { + if (snd_jack_new(a->card, sdev->uid, SND_JACK_HEADPHONE, &sdev->jack, true, false)) + dev_warn(a->dev, "Failed to create an attached jack for %s\n", sdev->uid); + aaudio_cmd_property_listener(a, sdev->dev_id, sdev->dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_JACK_PLUGGED, 0)); + aaudio_handle_jack_connection_change(sdev); + } + + aaudio_reply_free(&buf); + + list_add_tail(&sdev->list, &a->subdevice_list); + return; + +fail: + aaudio_reply_free(&buf); + kfree(sdev); +} + +static void aaudio_init_stream_info(struct aaudio_subdevice *sdev, struct aaudio_stream *strm) +{ + if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_PHYS_FORMAT, 0), NULL, 0, + &strm->desc, sizeof(strm->desc))) + dev_warn(sdev->a->dev, "Failed to query stream descriptor\n"); + if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_LATENCY, 0), NULL, 0, &strm->latency, sizeof(u32))) + dev_warn(sdev->a->dev, "Failed to query stream latency\n"); + if (strm->desc.format_id == AAUDIO_FORMAT_LPCM) + sdev->is_pcm = true; +} + +static void aaudio_free_dev(struct aaudio_subdevice *sdev) +{ + size_t i; + for (i = 0; i < sdev->in_stream_cnt; i++) { + if (sdev->in_streams[i].alsa_hw_desc) + kfree(sdev->in_streams[i].alsa_hw_desc); + if (sdev->in_streams[i].buffers) + kfree(sdev->in_streams[i].buffers); + } + for (i = 0; i < sdev->out_stream_cnt; i++) { + if (sdev->out_streams[i].alsa_hw_desc) + kfree(sdev->out_streams[i].alsa_hw_desc); + if (sdev->out_streams[i].buffers) + kfree(sdev->out_streams[i].buffers); + } + kfree(sdev); +} + +static struct aaudio_subdevice *aaudio_find_dev_by_dev_id(struct aaudio_device *a, aaudio_device_id_t dev_id) +{ + struct aaudio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (dev_id == sdev->dev_id) + return sdev; + } + return NULL; +} + +static struct aaudio_subdevice *aaudio_find_dev_by_uid(struct aaudio_device *a, const char *uid) +{ + struct aaudio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (!strcmp(uid, sdev->uid)) + return sdev; + } + return NULL; +} + +static void aaudio_init_bs_stream(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm); +static void aaudio_init_bs_stream_host(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm); + +static int aaudio_init_bs(struct aaudio_device *a) +{ + int i, j; + struct aaudio_buffer_struct_device *dev; + struct aaudio_subdevice *sdev; + u32 ver, sig, bs_base; + + ver = ioread32(&a->reg_mem_gpr[0]); + if (ver < 3) { + dev_err(a->dev, "aaudio: Bad GPR version (%u)", ver); + return -EINVAL; + } + sig = ioread32(&a->reg_mem_gpr[1]); + if (sig != AAUDIO_SIG) { + dev_err(a->dev, "aaudio: Bad GPR sig (%x)", sig); + return -EINVAL; + } + bs_base = ioread32(&a->reg_mem_gpr[2]); + a->bs = (struct aaudio_buffer_struct *) ((u8 *) a->reg_mem_bs + bs_base); + if (a->bs->signature != AAUDIO_SIG) { + dev_err(a->dev, "aaudio: Bad BufferStruct sig (%x)", a->bs->signature); + return -EINVAL; + } + dev_info(a->dev, "aaudio: BufferStruct ver = %i\n", a->bs->version); + dev_info(a->dev, "aaudio: Num devices = %i\n", a->bs->num_devices); + for (i = 0; i < a->bs->num_devices; i++) { + dev = &a->bs->devices[i]; + dev_info(a->dev, "aaudio: Device %i %s\n", i, dev->name); + + sdev = aaudio_find_dev_by_uid(a, dev->name); + if (!sdev) { + dev_err(a->dev, "aaudio: Subdevice not found for BufferStruct device %s\n", dev->name); + continue; + } + sdev->buf_id = (u8) i; + dev->num_input_streams = 0; + for (j = 0; j < dev->num_output_streams; j++) { + dev_info(a->dev, "aaudio: Device %i Stream %i: Output; Buffer Count = %i\n", i, j, + dev->output_streams[j].num_buffers); + if (j < sdev->out_stream_cnt) + aaudio_init_bs_stream(a, &sdev->out_streams[j], &dev->output_streams[j]); + } + } + + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->buf_id != AAUDIO_BUFFER_ID_NONE) + continue; + sdev->buf_id = i; + dev_info(a->dev, "aaudio: Created device %i %s\n", i, sdev->uid); + strcpy(a->bs->devices[i].name, sdev->uid); + a->bs->devices[i].num_input_streams = 0; + a->bs->devices[i].num_output_streams = 0; + a->bs->num_devices = ++i; + } + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->in_stream_cnt == 1) { + dev_info(a->dev, "aaudio: Device %i Host Stream; Input\n", sdev->buf_id); + aaudio_init_bs_stream_host(a, &sdev->in_streams[0], &a->bs->devices[sdev->buf_id].input_streams[0]); + a->bs->devices[sdev->buf_id].num_input_streams = 1; + wmb(); + + if (aaudio_cmd_set_input_stream_address_ranges(a, sdev->dev_id)) { + dev_err(a->dev, "aaudio: Failed to set input stream address ranges\n"); + } + } + } + + return 0; +} + +static void aaudio_init_bs_stream(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm) +{ + size_t i; + strm->buffer_cnt = bs_strm->num_buffers; + if (bs_strm->num_buffers > AAUDIO_DEIVCE_MAX_BUFFER_COUNT) { + dev_warn(a->dev, "BufferStruct buffer count %u exceeds driver limit of %u\n", bs_strm->num_buffers, + AAUDIO_DEIVCE_MAX_BUFFER_COUNT); + strm->buffer_cnt = AAUDIO_DEIVCE_MAX_BUFFER_COUNT; + } + if (!strm->buffer_cnt) + return; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct aaudio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + for (i = 0; i < strm->buffer_cnt; i++) { + strm->buffers[i].dma_addr = a->reg_mem_bs_dma + (dma_addr_t) bs_strm->buffers[i].address; + strm->buffers[i].ptr = a->reg_mem_bs + bs_strm->buffers[i].address; + strm->buffers[i].size = bs_strm->buffers[i].size; + } + + if (strm->buffer_cnt == 1) { + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (aaudio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } + } +} + +static void aaudio_init_bs_stream_host(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm) +{ + size_t size; + dma_addr_t dma_addr; + void *dma_ptr; + size = strm->desc.bytes_per_packet * 16640; + dma_ptr = dma_alloc_coherent(&a->pci->dev, size, &dma_addr, GFP_KERNEL); + if (!dma_ptr) { + dev_err(a->dev, "dma_alloc_coherent failed\n"); + return; + } + bs_strm->buffers[0].address = dma_addr; + bs_strm->buffers[0].size = size; + bs_strm->num_buffers = 1; + + memset(dma_ptr, 0, size); + + strm->buffer_cnt = 1; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct aaudio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + strm->buffers[0].dma_addr = dma_addr; + strm->buffers[0].ptr = dma_ptr; + strm->buffers[0].size = size; + + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (aaudio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } +} + +static void aaudio_handle_prop_change(struct aaudio_device *a, struct aaudio_msg *msg); + +void aaudio_handle_notification(struct aaudio_device *a, struct aaudio_msg *msg) +{ + struct aaudio_send_ctx sctx; + struct aaudio_msg_base base; + if (aaudio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case AAUDIO_MSG_NOTIFICATION_BOOT: + dev_info(a->dev, "Received boot notification from remote\n"); + + /* Resend the alive notify */ + if (aaudio_send(a, &sctx, 500, + aaudio_msg_write_alive_notification, 1, 3)) { + pr_err("Sending alive notification failed\n"); + } + break; + case AAUDIO_MSG_NOTIFICATION_ALIVE: + dev_info(a->dev, "Received alive notification from remote\n"); + complete_all(&a->remote_alive); + break; + case AAUDIO_MSG_PROPERTY_CHANGED: + aaudio_handle_prop_change(a, msg); + break; + default: + dev_info(a->dev, "Unhandled notification %i", base.msg); + break; + } +} + +struct aaudio_prop_change_work_struct { + struct work_struct ws; + struct aaudio_device *a; + aaudio_device_id_t dev; + aaudio_object_id_t obj; + struct aaudio_prop_addr prop; +}; + +static void aaudio_handle_jack_connection_change(struct aaudio_subdevice *sdev) +{ + u32 plugged; + if (!sdev->jack) + return; + /* NOTE: Apple made the plug status scoped to the input and output streams. This makes no sense for us, so I just + * always pick the OUTPUT status. */ + if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, sdev->dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_JACK_PLUGGED, 0), NULL, 0, &plugged, sizeof(plugged))) { + dev_err(sdev->a->dev, "Failed to get jack enable status\n"); + return; + } + dev_dbg(sdev->a->dev, "Jack is now %s\n", plugged ? "plugged" : "unplugged"); + snd_jack_report(sdev->jack, plugged ? sdev->jack->type : 0); +} + +void aaudio_handle_prop_change_work(struct work_struct *ws) +{ + struct aaudio_prop_change_work_struct *work = container_of(ws, struct aaudio_prop_change_work_struct, ws); + struct aaudio_subdevice *sdev; + + sdev = aaudio_find_dev_by_dev_id(work->a, work->dev); + if (!sdev) { + dev_err(work->a->dev, "Property notification change: device not found\n"); + goto done; + } + dev_dbg(work->a->dev, "Property changed for device: %s\n", sdev->uid); + + if (work->prop.scope == AAUDIO_PROP_SCOPE_OUTPUT && work->prop.selector == AAUDIO_PROP_JACK_PLUGGED) { + aaudio_handle_jack_connection_change(sdev); + } + +done: + kfree(work); +} + +void aaudio_handle_prop_change(struct aaudio_device *a, struct aaudio_msg *msg) +{ + /* NOTE: This is a scheduled work because this callback will generally need to query device information and this + * is not possible when we are in the reply parsing code's context. */ + struct aaudio_prop_change_work_struct *work; + work = kmalloc(sizeof(struct aaudio_prop_change_work_struct), GFP_KERNEL); + work->a = a; + INIT_WORK(&work->ws, aaudio_handle_prop_change_work); + aaudio_msg_read_property_changed(msg, &work->dev, &work->obj, &work->prop); + schedule_work(&work->ws); +} + +#define aaudio_send_cmd_response(a, sctx, msg, fn, ...) \ + if (aaudio_send_with_tag(a, sctx, ((struct aaudio_msg_header *) msg->data)->tag, 500, fn, ##__VA_ARGS__)) \ + pr_err("aaudio: Failed to reply to a command\n"); + +void aaudio_handle_cmd_timestamp(struct aaudio_device *a, struct aaudio_msg *msg) +{ + ktime_t time_os = ktime_get_boottime(); + struct aaudio_send_ctx sctx; + struct aaudio_subdevice *sdev; + u64 devid, timestamp, update_seed; + aaudio_msg_read_update_timestamp(msg, &devid, ×tamp, &update_seed); + dev_dbg(a->dev, "Received timestamp update for dev=%llx ts=%llx seed=%llx\n", devid, timestamp, update_seed); + + sdev = aaudio_find_dev_by_dev_id(a, devid); + aaudio_handle_timestamp(sdev, time_os, timestamp); + + aaudio_send_cmd_response(a, &sctx, msg, + aaudio_msg_write_update_timestamp_response); +} + +void aaudio_handle_command(struct aaudio_device *a, struct aaudio_msg *msg) +{ + struct aaudio_msg_base base; + if (aaudio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case AAUDIO_MSG_UPDATE_TIMESTAMP: + aaudio_handle_cmd_timestamp(a, msg); + break; + default: + dev_info(a->dev, "Unhandled device command %i", base.msg); + break; + } +} + +static struct pci_device_id aaudio_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1803) }, + { 0, }, +}; + +struct dev_pm_ops aaudio_pci_driver_pm = { + .suspend = aaudio_suspend, + .resume = aaudio_resume +}; +struct pci_driver aaudio_pci_driver = { + .name = "aaudio", + .id_table = aaudio_ids, + .probe = aaudio_probe, + .remove = aaudio_remove, + .driver = { + .pm = &aaudio_pci_driver_pm + } +}; + + +int aaudio_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&aaudio_chrdev, 0, 1, "aaudio"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + aaudio_class = class_create(THIS_MODULE, "aaudio"); +#else + aaudio_class = class_create("aaudio"); +#endif + if (IS_ERR(aaudio_class)) { + result = PTR_ERR(aaudio_class); + goto fail_class; + } + + result = pci_register_driver(&aaudio_pci_driver); + if (result) + goto fail_drv; + return 0; + +fail_drv: + pci_unregister_driver(&aaudio_pci_driver); +fail_class: + class_destroy(aaudio_class); +fail_chrdev: + unregister_chrdev_region(aaudio_chrdev, 1); + if (!result) + result = -EINVAL; + return result; +} + +void aaudio_module_exit(void) +{ + pci_unregister_driver(&aaudio_pci_driver); + class_destroy(aaudio_class); + unregister_chrdev_region(aaudio_chrdev, 1); +} + +struct aaudio_alsa_pcm_id_mapping aaudio_alsa_id_mappings[] = { + {"Speaker", 0}, + {"Digital Mic", 1}, + {"Codec Output", 2}, + {"Codec Input", 3}, + {"Bridge Loopback", 4}, + {} +}; + +module_param_named(index, aaudio_alsa_index, int, 0444); +MODULE_PARM_DESC(index, "Index value for Apple Internal Audio soundcard."); +module_param_named(id, aaudio_alsa_id, charp, 0444); +MODULE_PARM_DESC(id, "ID string for Apple Internal Audio soundcard."); diff --git a/drivers/staging/apple-bce/audio/audio.h b/drivers/staging/apple-bce/audio/audio.h new file mode 100644 index 000000000000..004bc1e22ea4 --- /dev/null +++ b/drivers/staging/apple-bce/audio/audio.h @@ -0,0 +1,125 @@ +#ifndef AAUDIO_H +#define AAUDIO_H + +#include +#include +#include "../apple_bce.h" +#include "protocol_bce.h" +#include "description.h" + +#define AAUDIO_SIG 0x19870423 + +#define AAUDIO_DEVICE_MAX_UID_LEN 128 +#define AAUDIO_DEIVCE_MAX_INPUT_STREAMS 1 +#define AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS 1 +#define AAUDIO_DEIVCE_MAX_BUFFER_COUNT 1 + +#define AAUDIO_BUFFER_ID_NONE 0xffu + +struct snd_card; +struct snd_pcm; +struct snd_pcm_hardware; +struct snd_jack; + +struct __attribute__((packed)) __attribute__((aligned(4))) aaudio_buffer_struct_buffer { + size_t address; + size_t size; + size_t pad[4]; +}; +struct aaudio_buffer_struct_stream { + u8 num_buffers; + struct aaudio_buffer_struct_buffer buffers[100]; + char filler[32]; +}; +struct aaudio_buffer_struct_device { + char name[128]; + u8 num_input_streams; + u8 num_output_streams; + struct aaudio_buffer_struct_stream input_streams[5]; + struct aaudio_buffer_struct_stream output_streams[5]; + char filler[128]; +}; +struct aaudio_buffer_struct { + u32 version; + u32 signature; + u32 flags; + u8 num_devices; + struct aaudio_buffer_struct_device devices[20]; +}; + +struct aaudio_device; +struct aaudio_dma_buf { + dma_addr_t dma_addr; + void *ptr; + size_t size; +}; +struct aaudio_stream { + aaudio_object_id_t id; + size_t buffer_cnt; + struct aaudio_dma_buf *buffers; + + struct aaudio_apple_description desc; + struct snd_pcm_hardware *alsa_hw_desc; + u32 latency; + + bool waiting_for_first_ts; + + ktime_t remote_timestamp; + snd_pcm_sframes_t frame_min; + int started; +}; +struct aaudio_subdevice { + struct aaudio_device *a; + struct list_head list; + aaudio_device_id_t dev_id; + u32 in_latency, out_latency; + u8 buf_id; + int alsa_id; + char uid[AAUDIO_DEVICE_MAX_UID_LEN + 1]; + size_t in_stream_cnt; + struct aaudio_stream in_streams[AAUDIO_DEIVCE_MAX_INPUT_STREAMS]; + size_t out_stream_cnt; + struct aaudio_stream out_streams[AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS]; + bool is_pcm; + struct snd_pcm *pcm; + struct snd_jack *jack; +}; +struct aaudio_alsa_pcm_id_mapping { + const char *name; + int alsa_id; +}; + +struct aaudio_device { + struct pci_dev *pci; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_bs; + dma_addr_t reg_mem_bs_dma; + void __iomem *reg_mem_cfg; + + u32 __iomem *reg_mem_gpr; + + struct aaudio_buffer_struct *bs; + + struct apple_bce_device *bce; + struct aaudio_bce bcem; + + struct snd_card *card; + + struct list_head subdevice_list; + int next_alsa_id; + + struct completion remote_alive; +}; + +void aaudio_handle_notification(struct aaudio_device *a, struct aaudio_msg *msg); +void aaudio_handle_prop_change_work(struct work_struct *ws); +void aaudio_handle_cmd_timestamp(struct aaudio_device *a, struct aaudio_msg *msg); +void aaudio_handle_command(struct aaudio_device *a, struct aaudio_msg *msg); + +int aaudio_module_init(void); +void aaudio_module_exit(void); + +extern struct aaudio_alsa_pcm_id_mapping aaudio_alsa_id_mappings[]; + +#endif //AAUDIO_H diff --git a/drivers/staging/apple-bce/audio/description.h b/drivers/staging/apple-bce/audio/description.h new file mode 100644 index 000000000000..dfef3ab68f27 --- /dev/null +++ b/drivers/staging/apple-bce/audio/description.h @@ -0,0 +1,42 @@ +#ifndef AAUDIO_DESCRIPTION_H +#define AAUDIO_DESCRIPTION_H + +#include + +struct aaudio_apple_description { + u64 sample_rate_double; + u32 format_id; + u32 format_flags; + u32 bytes_per_packet; + u32 frames_per_packet; + u32 bytes_per_frame; + u32 channels_per_frame; + u32 bits_per_channel; + u32 reserved; +}; + +enum { + AAUDIO_FORMAT_LPCM = 0x6c70636d // 'lpcm' +}; + +enum { + AAUDIO_FORMAT_FLAG_FLOAT = 1, + AAUDIO_FORMAT_FLAG_BIG_ENDIAN = 2, + AAUDIO_FORMAT_FLAG_SIGNED = 4, + AAUDIO_FORMAT_FLAG_PACKED = 8, + AAUDIO_FORMAT_FLAG_ALIGNED_HIGH = 16, + AAUDIO_FORMAT_FLAG_NON_INTERLEAVED = 32, + AAUDIO_FORMAT_FLAG_NON_MIXABLE = 64 +}; + +static inline u64 aaudio_double_to_u64(u64 d) +{ + u8 sign = (u8) ((d >> 63) & 1); + s32 exp = (s32) ((d >> 52) & 0x7ff) - 1023; + u64 fr = d & ((1LL << 52) - 1); + if (sign || exp < 0) + return 0; + return (u64) ((1LL << exp) + (fr >> (52 - exp))); +} + +#endif //AAUDIO_DESCRIPTION_H diff --git a/drivers/staging/apple-bce/audio/pcm.c b/drivers/staging/apple-bce/audio/pcm.c new file mode 100644 index 000000000000..1026e10a9ac5 --- /dev/null +++ b/drivers/staging/apple-bce/audio/pcm.c @@ -0,0 +1,308 @@ +#include "pcm.h" +#include "audio.h" + +static u64 aaudio_get_alsa_fmtbit(struct aaudio_apple_description *desc) +{ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_FLOAT) { + if (desc->bits_per_channel == 32) { + if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT_LE; + } else if (desc->bits_per_channel == 64) { + if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT64_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT64_LE; + } else { + pr_err("aaudio: unsupported bits per channel for float format: %u\n", desc->bits_per_channel); + return 0; + } + } +#define DEFINE_BPC_OPTION(val, b) \ + case val: \ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) { \ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## BE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## BE; \ + } else { \ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## LE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## LE; \ + } + if (desc->format_flags & AAUDIO_FORMAT_FLAG_PACKED) { + switch (desc->bits_per_channel) { + case 8: + case 16: + case 32: + break; + DEFINE_BPC_OPTION(24, 24_3) + default: + pr_err("aaudio: unsupported bits per channel for packed format: %u\n", desc->bits_per_channel); + return 0; + } + } + if (desc->format_flags & AAUDIO_FORMAT_FLAG_ALIGNED_HIGH) { + switch (desc->bits_per_channel) { + DEFINE_BPC_OPTION(24, 32_) + default: + pr_err("aaudio: unsupported bits per channel for high-aligned format: %u\n", desc->bits_per_channel); + return 0; + } + } + switch (desc->bits_per_channel) { + case 8: + if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) + return SNDRV_PCM_FMTBIT_S8; + else + return SNDRV_PCM_FMTBIT_U8; + DEFINE_BPC_OPTION(16, 16_) + DEFINE_BPC_OPTION(24, 24_) + DEFINE_BPC_OPTION(32, 32_) + default: + pr_err("aaudio: unsupported bits per channel: %u\n", desc->bits_per_channel); + return 0; + } +} +int aaudio_create_hw_info(struct aaudio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, + size_t buf_size) +{ + uint rate; + alsa_hw->info = (SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_BLOCK_TRANSFER | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_DOUBLE); + if (desc->format_flags & AAUDIO_FORMAT_FLAG_NON_MIXABLE) + pr_warn("aaudio: unsupported hw flag: NON_MIXABLE\n"); + if (!(desc->format_flags & AAUDIO_FORMAT_FLAG_NON_INTERLEAVED)) + alsa_hw->info |= SNDRV_PCM_INFO_INTERLEAVED; + alsa_hw->formats = aaudio_get_alsa_fmtbit(desc); + if (!alsa_hw->formats) + return -EINVAL; + rate = (uint) aaudio_double_to_u64(desc->sample_rate_double); + alsa_hw->rates = snd_pcm_rate_to_rate_bit(rate); + alsa_hw->rate_min = rate; + alsa_hw->rate_max = rate; + alsa_hw->channels_min = desc->channels_per_frame; + alsa_hw->channels_max = desc->channels_per_frame; + alsa_hw->buffer_bytes_max = buf_size; + alsa_hw->period_bytes_min = desc->bytes_per_packet; + alsa_hw->period_bytes_max = desc->bytes_per_packet; + alsa_hw->periods_min = (uint) (buf_size / desc->bytes_per_packet); + alsa_hw->periods_max = (uint) (buf_size / desc->bytes_per_packet); + pr_debug("aaudio_create_hw_info: format = %llu, rate = %u/%u. channels = %u, periods = %u, period size = %lu\n", + alsa_hw->formats, alsa_hw->rate_min, alsa_hw->rates, alsa_hw->channels_min, alsa_hw->periods_min, + alsa_hw->period_bytes_min); + return 0; +} + +static struct aaudio_stream *aaudio_pcm_stream(struct snd_pcm_substream *substream) +{ + struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + return &sdev->out_streams[substream->number]; + else + return &sdev->in_streams[substream->number]; +} + +static int aaudio_pcm_open(struct snd_pcm_substream *substream) +{ + pr_debug("aaudio_pcm_open\n"); + substream->runtime->hw = *aaudio_pcm_stream(substream)->alsa_hw_desc; + + return 0; +} + +static int aaudio_pcm_close(struct snd_pcm_substream *substream) +{ + pr_debug("aaudio_pcm_close\n"); + return 0; +} + +static int aaudio_pcm_prepare(struct snd_pcm_substream *substream) +{ + return 0; +} + +static int aaudio_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) +{ + struct aaudio_stream *astream = aaudio_pcm_stream(substream); + pr_debug("aaudio_pcm_hw_params\n"); + + if (!astream->buffer_cnt || !astream->buffers) + return -EINVAL; + + substream->runtime->dma_area = astream->buffers[0].ptr; + substream->runtime->dma_addr = astream->buffers[0].dma_addr; + substream->runtime->dma_bytes = astream->buffers[0].size; + return 0; +} + +static int aaudio_pcm_hw_free(struct snd_pcm_substream *substream) +{ + pr_debug("aaudio_pcm_hw_free\n"); + return 0; +} + +static void aaudio_pcm_start(struct snd_pcm_substream *substream) +{ + struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct aaudio_stream *stream = aaudio_pcm_stream(substream); + void *buf; + size_t s; + ktime_t time_start, time_end; + bool back_buffer; + time_start = ktime_get(); + + back_buffer = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); + + if (back_buffer) { + s = frames_to_bytes(substream->runtime, substream->runtime->control->appl_ptr); + buf = kmalloc(s, GFP_KERNEL); + memcpy_fromio(buf, substream->runtime->dma_area, s); + time_end = ktime_get(); + pr_debug("aaudio: Backed up the buffer in %lluns [%li]\n", ktime_to_ns(time_end - time_start), + substream->runtime->control->appl_ptr); + } + + stream->waiting_for_first_ts = true; + stream->frame_min = stream->latency; + + aaudio_cmd_start_io(sdev->a, sdev->dev_id); + if (back_buffer) + memcpy_toio(substream->runtime->dma_area, buf, s); + + time_end = ktime_get(); + pr_debug("aaudio: Started the audio device in %lluns\n", ktime_to_ns(time_end - time_start)); +} + +static int aaudio_pcm_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct aaudio_stream *stream = aaudio_pcm_stream(substream); + pr_debug("aaudio_pcm_trigger %x\n", cmd); + + /* We only supports triggers on the #0 buffer */ + if (substream->number != 0) + return 0; + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + aaudio_pcm_start(substream); + stream->started = 1; + break; + case SNDRV_PCM_TRIGGER_STOP: + aaudio_cmd_stop_io(sdev->a, sdev->dev_id); + stream->started = 0; + break; + default: + return -EINVAL; + } + return 0; +} + +static snd_pcm_uframes_t aaudio_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct aaudio_stream *stream = aaudio_pcm_stream(substream); + ktime_t time_from_start; + snd_pcm_sframes_t frames; + snd_pcm_sframes_t buffer_time_length; + + if (!stream->started || stream->waiting_for_first_ts) { + pr_warn("aaudio_pcm_pointer while not started\n"); + return 0; + } + + /* Approximate the pointer based on the last received timestamp */ + time_from_start = ktime_get_boottime() - stream->remote_timestamp; + buffer_time_length = NSEC_PER_SEC * substream->runtime->buffer_size / substream->runtime->rate; + frames = (ktime_to_ns(time_from_start) % buffer_time_length) * substream->runtime->buffer_size / buffer_time_length; + if (ktime_to_ns(time_from_start) < buffer_time_length) { + if (frames < stream->frame_min) + frames = stream->frame_min; + else + stream->frame_min = 0; + } else { + if (ktime_to_ns(time_from_start) < 2 * buffer_time_length) + stream->frame_min = frames; + else + stream->frame_min = 0; /* Heavy desync */ + } + frames -= stream->latency; + if (frames < 0) + frames += ((-frames - 1) / substream->runtime->buffer_size + 1) * substream->runtime->buffer_size; + return (snd_pcm_uframes_t) frames; +} + +static struct snd_pcm_ops aaudio_pcm_ops = { + .open = aaudio_pcm_open, + .close = aaudio_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = aaudio_pcm_hw_params, + .hw_free = aaudio_pcm_hw_free, + .prepare = aaudio_pcm_prepare, + .trigger = aaudio_pcm_trigger, + .pointer = aaudio_pcm_pointer, + .mmap = snd_pcm_lib_mmap_iomem +}; + +int aaudio_create_pcm(struct aaudio_subdevice *sdev) +{ + struct snd_pcm *pcm; + struct aaudio_alsa_pcm_id_mapping *id_mapping; + int err; + + if (!sdev->is_pcm || (sdev->in_stream_cnt == 0 && sdev->out_stream_cnt == 0)) { + return -EINVAL; + } + + for (id_mapping = aaudio_alsa_id_mappings; id_mapping->name; id_mapping++) { + if (!strcmp(sdev->uid, id_mapping->name)) { + sdev->alsa_id = id_mapping->alsa_id; + break; + } + } + if (!id_mapping->name) + sdev->alsa_id = sdev->a->next_alsa_id++; + err = snd_pcm_new(sdev->a->card, sdev->uid, sdev->alsa_id, + (int) sdev->out_stream_cnt, (int) sdev->in_stream_cnt, &pcm); + if (err < 0) + return err; + pcm->private_data = sdev; + pcm->nonatomic = 1; + sdev->pcm = pcm; + strcpy(pcm->name, sdev->uid); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &aaudio_pcm_ops); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &aaudio_pcm_ops); + return 0; +} + +static void aaudio_handle_stream_timestamp(struct snd_pcm_substream *substream, ktime_t timestamp) +{ + unsigned long flags; + struct aaudio_stream *stream; + + stream = aaudio_pcm_stream(substream); + snd_pcm_stream_lock_irqsave(substream, flags); + stream->remote_timestamp = timestamp; + if (stream->waiting_for_first_ts) { + stream->waiting_for_first_ts = false; + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + snd_pcm_stream_unlock_irqrestore(substream, flags); + snd_pcm_period_elapsed(substream); +} + +void aaudio_handle_timestamp(struct aaudio_subdevice *sdev, ktime_t os_timestamp, u64 dev_timestamp) +{ + struct snd_pcm_substream *substream; + + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (substream) + aaudio_handle_stream_timestamp(substream, dev_timestamp); + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + if (substream) + aaudio_handle_stream_timestamp(substream, os_timestamp); +} diff --git a/drivers/staging/apple-bce/audio/pcm.h b/drivers/staging/apple-bce/audio/pcm.h new file mode 100644 index 000000000000..ea5f35fbe408 --- /dev/null +++ b/drivers/staging/apple-bce/audio/pcm.h @@ -0,0 +1,16 @@ +#ifndef AAUDIO_PCM_H +#define AAUDIO_PCM_H + +#include +#include + +struct aaudio_subdevice; +struct aaudio_apple_description; +struct snd_pcm_hardware; + +int aaudio_create_hw_info(struct aaudio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, size_t buf_size); +int aaudio_create_pcm(struct aaudio_subdevice *sdev); + +void aaudio_handle_timestamp(struct aaudio_subdevice *sdev, ktime_t os_timestamp, u64 dev_timestamp); + +#endif //AAUDIO_PCM_H diff --git a/drivers/staging/apple-bce/audio/protocol.c b/drivers/staging/apple-bce/audio/protocol.c new file mode 100644 index 000000000000..2314813aeead --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol.c @@ -0,0 +1,347 @@ +#include "protocol.h" +#include "protocol_bce.h" +#include "audio.h" + +int aaudio_msg_read_base(struct aaudio_msg *msg, struct aaudio_msg_base *base) +{ + if (msg->size < sizeof(struct aaudio_msg_header) + sizeof(struct aaudio_msg_base) * 2) + return -EINVAL; + *base = *((struct aaudio_msg_base *) ((struct aaudio_msg_header *) msg->data + 1)); + return 0; +} + +#define READ_START(type) \ + size_t offset = sizeof(struct aaudio_msg_header) + sizeof(struct aaudio_msg_base); (void)offset; \ + if (((struct aaudio_msg_base *) ((struct aaudio_msg_header *) msg->data + 1))->msg != type) \ + return -EINVAL; +#define READ_DEVID_VAR(devid) *devid = ((struct aaudio_msg_header *) msg->data)->device_id +#define READ_VAL(type) ({ offset += sizeof(type); *((type *) ((u8 *) msg->data + offset - sizeof(type))); }) +#define READ_VAR(type, var) *var = READ_VAL(type) + +int aaudio_msg_read_start_io_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_START_IO_RESPONSE); + return 0; +} + +int aaudio_msg_read_stop_io_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_STOP_IO_RESPONSE); + return 0; +} + +int aaudio_msg_read_update_timestamp(struct aaudio_msg *msg, aaudio_device_id_t *devid, + u64 *timestamp, u64 *update_seed) +{ + READ_START(AAUDIO_MSG_UPDATE_TIMESTAMP); + READ_DEVID_VAR(devid); + READ_VAR(u64, timestamp); + READ_VAR(u64, update_seed); + return 0; +} + +int aaudio_msg_read_get_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop, void **data, u64 *data_size) +{ + READ_START(AAUDIO_MSG_GET_PROPERTY_RESPONSE); + READ_VAR(aaudio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + READ_VAR(u64, data_size); + *data = ((u8 *) msg->data + offset); + /* offset += data_size; */ + return 0; +} + +int aaudio_msg_read_set_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj) +{ + READ_START(AAUDIO_MSG_SET_PROPERTY_RESPONSE); + READ_VAR(aaudio_object_id_t, obj); + return 0; +} + +int aaudio_msg_read_property_listener_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop) +{ + READ_START(AAUDIO_MSG_PROPERTY_LISTENER_RESPONSE); + READ_VAR(aaudio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int aaudio_msg_read_property_changed(struct aaudio_msg *msg, aaudio_device_id_t *devid, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop) +{ + READ_START(AAUDIO_MSG_PROPERTY_CHANGED); + READ_DEVID_VAR(devid); + READ_VAR(aaudio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int aaudio_msg_read_set_input_stream_address_ranges_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE); + return 0; +} + +int aaudio_msg_read_get_input_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(AAUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(aaudio_object_id_t); */ + return 0; +} + +int aaudio_msg_read_get_output_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(AAUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(aaudio_object_id_t); */ + return 0; +} + +int aaudio_msg_read_set_remote_access_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE); + return 0; +} + +int aaudio_msg_read_get_device_list_response(struct aaudio_msg *msg, aaudio_device_id_t **dev_l, u64 *dev_cnt) +{ + READ_START(AAUDIO_MSG_GET_DEVICE_LIST_RESPONSE); + READ_VAR(u64, dev_cnt); + *dev_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += dev_cnt * sizeof(aaudio_device_id_t); */ + return 0; +} + +#define WRITE_START_OF_TYPE(typev, devid) \ + size_t offset = sizeof(struct aaudio_msg_header); (void) offset; \ + ((struct aaudio_msg_header *) msg->data)->type = (typev); \ + ((struct aaudio_msg_header *) msg->data)->device_id = (devid); +#define WRITE_START_COMMAND(devid) WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_COMMAND, devid) +#define WRITE_START_RESPONSE() WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_RESPONSE, 0) +#define WRITE_START_NOTIFICATION() WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_NOTIFICATION, 0) +#define WRITE_VAL(type, value) { *((type *) ((u8 *) msg->data + offset)) = value; offset += sizeof(value); } +#define WRITE_BIN(value, size) { memcpy((u8 *) msg->data + offset, value, size); offset += size; } +#define WRITE_BASE(type) WRITE_VAL(u32, type) WRITE_VAL(u32, 0) +#define WRITE_END() { msg->size = offset; } + +void aaudio_msg_write_start_io(struct aaudio_msg *msg, aaudio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_START_IO); + WRITE_END(); +} + +void aaudio_msg_write_stop_io(struct aaudio_msg *msg, aaudio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_STOP_IO); + WRITE_END(); +} + +void aaudio_msg_write_get_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_GET_PROPERTY); + WRITE_VAL(aaudio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void aaudio_msg_write_set_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_SET_PROPERTY); + WRITE_VAL(aaudio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, data_size); + WRITE_BIN(data, data_size); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void aaudio_msg_write_property_listener(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_PROPERTY_LISTENER); + WRITE_VAL(aaudio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_END(); +} + +void aaudio_msg_write_set_input_stream_address_ranges(struct aaudio_msg *msg, aaudio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES); + WRITE_END(); +} + +void aaudio_msg_write_get_input_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(AAUDIO_MSG_GET_INPUT_STREAM_LIST); + WRITE_END(); +} + +void aaudio_msg_write_get_output_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(AAUDIO_MSG_GET_OUTPUT_STREAM_LIST); + WRITE_END(); +} + +void aaudio_msg_write_set_remote_access(struct aaudio_msg *msg, u64 mode) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(AAUDIO_MSG_SET_REMOTE_ACCESS); + WRITE_VAL(u64, mode); + WRITE_END(); +} + +void aaudio_msg_write_alive_notification(struct aaudio_msg *msg, u32 proto_ver, u32 msg_ver) +{ + WRITE_START_NOTIFICATION(); + WRITE_BASE(AAUDIO_MSG_NOTIFICATION_ALIVE); + WRITE_VAL(u32, proto_ver); + WRITE_VAL(u32, msg_ver); + WRITE_END(); +} + +void aaudio_msg_write_update_timestamp_response(struct aaudio_msg *msg) +{ + WRITE_START_RESPONSE(); + WRITE_BASE(AAUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE); + WRITE_END(); +} + +void aaudio_msg_write_get_device_list(struct aaudio_msg *msg) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(AAUDIO_MSG_GET_DEVICE_LIST); + WRITE_END(); +} + +#define CMD_SHARED_VARS_NO_REPLY \ + int status = 0; \ + struct aaudio_send_ctx sctx; +#define CMD_SHARED_VARS \ + CMD_SHARED_VARS_NO_REPLY \ + struct aaudio_msg reply = aaudio_reply_alloc(); \ + struct aaudio_msg *buf = &reply; +#define CMD_SEND_REQUEST(fn, ...) \ + if ((status = aaudio_send_cmd_sync(a, &sctx, buf, 500, fn, ##__VA_ARGS__))) \ + return status; +#define CMD_DEF_SHARED_AND_SEND(fn, ...) \ + CMD_SHARED_VARS \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_DEF_SHARED_NO_REPLY_AND_SEND(fn, ...) \ + CMD_SHARED_VARS_NO_REPLY \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_HNDL_REPLY_NO_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + return status; +#define CMD_HNDL_REPLY_AND_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + aaudio_reply_free(&reply); \ + return status; + +int aaudio_cmd_start_io(struct aaudio_device *a, aaudio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_start_io, devid); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_start_io_response); +} +int aaudio_cmd_stop_io(struct aaudio_device *a, aaudio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_stop_io, devid); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_stop_io_response); +} +int aaudio_cmd_get_property(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_property, devid, obj, prop, qualifier, qualifier_size); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_property_response, &obj, &prop, data, data_size); +} +int aaudio_cmd_get_primitive_property(struct aaudio_device *a, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + int status; + struct aaudio_msg reply = aaudio_reply_alloc(); + void *r_data; + u64 r_data_size; + if ((status = aaudio_cmd_get_property(a, &reply, devid, obj, prop, qualifier, qualifier_size, + &r_data, &r_data_size))) + goto finish; + if (r_data_size != data_size) { + status = -EINVAL; + goto finish; + } + memcpy(data, r_data, data_size); +finish: + aaudio_reply_free(&reply); + return status; +} +int aaudio_cmd_set_property(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_property, devid, obj, prop, data, data_size, + qualifier, qualifier_size); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_property_response, &obj); +} +int aaudio_cmd_property_listener(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_property_listener, devid, obj, prop); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_property_listener_response, &obj, &prop); +} +int aaudio_cmd_set_input_stream_address_ranges(struct aaudio_device *a, aaudio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_input_stream_address_ranges, devid); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_input_stream_address_ranges_response); +} +int aaudio_cmd_get_input_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_input_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_input_stream_list_response, str_l, str_cnt); +} +int aaudio_cmd_get_output_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_output_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_output_stream_list_response, str_l, str_cnt); +} +int aaudio_cmd_set_remote_access(struct aaudio_device *a, u64 mode) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_remote_access, mode); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_remote_access_response); +} +int aaudio_cmd_get_device_list(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t **dev_l, u64 *dev_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_device_list); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_device_list_response, dev_l, dev_cnt); +} \ No newline at end of file diff --git a/drivers/staging/apple-bce/audio/protocol.h b/drivers/staging/apple-bce/audio/protocol.h new file mode 100644 index 000000000000..3427486f3f57 --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol.h @@ -0,0 +1,147 @@ +#ifndef AAUDIO_PROTOCOL_H +#define AAUDIO_PROTOCOL_H + +#include + +struct aaudio_device; + +typedef u64 aaudio_device_id_t; +typedef u64 aaudio_object_id_t; + +struct aaudio_msg { + void *data; + size_t size; +}; + +struct __attribute__((packed)) aaudio_msg_header { + char tag[4]; + u8 type; + aaudio_device_id_t device_id; // Idk, use zero for commands? +}; +struct __attribute__((packed)) aaudio_msg_base { + u32 msg; + u32 status; +}; + +struct aaudio_prop_addr { + u32 scope; + u32 selector; + u32 element; +}; +#define AAUDIO_PROP(scope, sel, el) (struct aaudio_prop_addr) { scope, sel, el } + +enum { + AAUDIO_MSG_TYPE_COMMAND = 1, + AAUDIO_MSG_TYPE_RESPONSE = 2, + AAUDIO_MSG_TYPE_NOTIFICATION = 3 +}; + +enum { + AAUDIO_MSG_START_IO = 0, + AAUDIO_MSG_START_IO_RESPONSE = 1, + AAUDIO_MSG_STOP_IO = 2, + AAUDIO_MSG_STOP_IO_RESPONSE = 3, + AAUDIO_MSG_UPDATE_TIMESTAMP = 4, + AAUDIO_MSG_GET_PROPERTY = 7, + AAUDIO_MSG_GET_PROPERTY_RESPONSE = 8, + AAUDIO_MSG_SET_PROPERTY = 9, + AAUDIO_MSG_SET_PROPERTY_RESPONSE = 10, + AAUDIO_MSG_PROPERTY_LISTENER = 11, + AAUDIO_MSG_PROPERTY_LISTENER_RESPONSE = 12, + AAUDIO_MSG_PROPERTY_CHANGED = 13, + AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES = 18, + AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE = 19, + AAUDIO_MSG_GET_INPUT_STREAM_LIST = 24, + AAUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE = 25, + AAUDIO_MSG_GET_OUTPUT_STREAM_LIST = 26, + AAUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE = 27, + AAUDIO_MSG_SET_REMOTE_ACCESS = 32, + AAUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE = 33, + AAUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE = 34, + + AAUDIO_MSG_NOTIFICATION_ALIVE = 100, + AAUDIO_MSG_GET_DEVICE_LIST = 101, + AAUDIO_MSG_GET_DEVICE_LIST_RESPONSE = 102, + AAUDIO_MSG_NOTIFICATION_BOOT = 104 +}; + +enum { + AAUDIO_REMOTE_ACCESS_OFF = 0, + AAUDIO_REMOTE_ACCESS_ON = 2 +}; + +enum { + AAUDIO_PROP_SCOPE_GLOBAL = 0x676c6f62, // 'glob' + AAUDIO_PROP_SCOPE_INPUT = 0x696e7074, // 'inpt' + AAUDIO_PROP_SCOPE_OUTPUT = 0x6f757470 // 'outp' +}; + +enum { + AAUDIO_PROP_UID = 0x75696420, // 'uid ' + AAUDIO_PROP_BOOL_VALUE = 0x6263766c, // 'bcvl' + AAUDIO_PROP_JACK_PLUGGED = 0x6a61636b, // 'jack' + AAUDIO_PROP_SEL_VOLUME = 0x64656176, // 'deav' + AAUDIO_PROP_LATENCY = 0x6c746e63, // 'ltnc' + AAUDIO_PROP_PHYS_FORMAT = 0x70667420 // 'pft ' +}; + +int aaudio_msg_read_base(struct aaudio_msg *msg, struct aaudio_msg_base *base); + +int aaudio_msg_read_start_io_response(struct aaudio_msg *msg); +int aaudio_msg_read_stop_io_response(struct aaudio_msg *msg); +int aaudio_msg_read_update_timestamp(struct aaudio_msg *msg, aaudio_device_id_t *devid, + u64 *timestamp, u64 *update_seed); +int aaudio_msg_read_get_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop, void **data, u64 *data_size); +int aaudio_msg_read_set_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj); +int aaudio_msg_read_property_listener_response(struct aaudio_msg *msg,aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop); +int aaudio_msg_read_property_changed(struct aaudio_msg *msg, aaudio_device_id_t *devid, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop); +int aaudio_msg_read_set_input_stream_address_ranges_response(struct aaudio_msg *msg); +int aaudio_msg_read_get_input_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_msg_read_get_output_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_msg_read_set_remote_access_response(struct aaudio_msg *msg); +int aaudio_msg_read_get_device_list_response(struct aaudio_msg *msg, aaudio_device_id_t **dev_l, u64 *dev_cnt); + +void aaudio_msg_write_start_io(struct aaudio_msg *msg, aaudio_device_id_t dev); +void aaudio_msg_write_stop_io(struct aaudio_msg *msg, aaudio_device_id_t dev); +void aaudio_msg_write_get_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size); +void aaudio_msg_write_set_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size); +void aaudio_msg_write_property_listener(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop); +void aaudio_msg_write_set_input_stream_address_ranges(struct aaudio_msg *msg, aaudio_device_id_t devid); +void aaudio_msg_write_get_input_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid); +void aaudio_msg_write_get_output_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid); +void aaudio_msg_write_set_remote_access(struct aaudio_msg *msg, u64 mode); +void aaudio_msg_write_alive_notification(struct aaudio_msg *msg, u32 proto_ver, u32 msg_ver); +void aaudio_msg_write_update_timestamp_response(struct aaudio_msg *msg); +void aaudio_msg_write_get_device_list(struct aaudio_msg *msg); + + +int aaudio_cmd_start_io(struct aaudio_device *a, aaudio_device_id_t devid); +int aaudio_cmd_stop_io(struct aaudio_device *a, aaudio_device_id_t devid); +int aaudio_cmd_get_property(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size); +int aaudio_cmd_get_primitive_property(struct aaudio_device *a, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int aaudio_cmd_set_property(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int aaudio_cmd_property_listener(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop); +int aaudio_cmd_set_input_stream_address_ranges(struct aaudio_device *a, aaudio_device_id_t devid); +int aaudio_cmd_get_input_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_cmd_get_output_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_cmd_set_remote_access(struct aaudio_device *a, u64 mode); +int aaudio_cmd_get_device_list(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t **dev_l, u64 *dev_cnt); + + + +#endif //AAUDIO_PROTOCOL_H diff --git a/drivers/staging/apple-bce/audio/protocol_bce.c b/drivers/staging/apple-bce/audio/protocol_bce.c new file mode 100644 index 000000000000..28f2dfd44d67 --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol_bce.c @@ -0,0 +1,226 @@ +#include "protocol_bce.h" + +#include "audio.h" + +static void aaudio_bce_out_queue_completion(struct bce_queue_sq *sq); +static void aaudio_bce_in_queue_completion(struct bce_queue_sq *sq); +static int aaudio_bce_queue_init(struct aaudio_device *dev, struct aaudio_bce_queue *q, const char *name, int direction, + bce_sq_completion cfn); +void aaudio_bce_in_queue_submit_pending(struct aaudio_bce_queue *q, size_t count); + +int aaudio_bce_init(struct aaudio_device *dev) +{ + int status; + struct aaudio_bce *bce = &dev->bcem; + bce->cq = bce_create_cq(dev->bce, 0x80); + spin_lock_init(&bce->spinlock); + if (!bce->cq) + return -EINVAL; + if ((status = aaudio_bce_queue_init(dev, &bce->qout, "com.apple.BridgeAudio.IntelToARM", DMA_TO_DEVICE, + aaudio_bce_out_queue_completion))) { + return status; + } + if ((status = aaudio_bce_queue_init(dev, &bce->qin, "com.apple.BridgeAudio.ARMToIntel", DMA_FROM_DEVICE, + aaudio_bce_in_queue_completion))) { + return status; + } + aaudio_bce_in_queue_submit_pending(&bce->qin, bce->qin.el_count); + return 0; +} + +int aaudio_bce_queue_init(struct aaudio_device *dev, struct aaudio_bce_queue *q, const char *name, int direction, + bce_sq_completion cfn) +{ + q->cq = dev->bcem.cq; + q->el_size = AAUDIO_BCE_QUEUE_ELEMENT_SIZE; + q->el_count = AAUDIO_BCE_QUEUE_ELEMENT_COUNT; + /* NOTE: The Apple impl uses 0x80 as the queue size, however we use 21 (in fact 20) to simplify the impl */ + q->sq = bce_create_sq(dev->bce, q->cq, name, (u32) (q->el_count + 1), direction, cfn, dev); + if (!q->sq) + return -EINVAL; + + q->data = dma_alloc_coherent(&dev->bce->pci->dev, q->el_size * q->el_count, &q->dma_addr, GFP_KERNEL); + if (!q->data) { + bce_destroy_sq(dev->bce, q->sq); + return -EINVAL; + } + return 0; +} + +static void aaudio_send_create_tag(struct aaudio_bce *b, int *tagn, char tag[4]) +{ + char tag_zero[5]; + b->tag_num = (b->tag_num + 1) % AAUDIO_BCE_QUEUE_TAG_COUNT; + *tagn = b->tag_num; + snprintf(tag_zero, 5, "S%03d", b->tag_num); + *((u32 *) tag) = *((u32 *) tag_zero); +} + +int __aaudio_send_prepare(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, char *tag) +{ + int status; + size_t index; + void *dptr; + struct aaudio_msg_header *header; + if ((status = bce_reserve_submission(b->qout.sq, &ctx->timeout))) + return status; + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + index = b->qout.data_tail; + dptr = (u8 *) b->qout.data + index * b->qout.el_size; + ctx->msg.data = dptr; + header = dptr; + if (tag) + *((u32 *) header->tag) = *((u32 *) tag); + else + aaudio_send_create_tag(b, &ctx->tag_n, header->tag); + return 0; +} + +void __aaudio_send(struct aaudio_bce *b, struct aaudio_send_ctx *ctx) +{ + struct bce_qe_submission *s = bce_next_submission(b->qout.sq); +#ifdef DEBUG + pr_debug("aaudio: Sending command data\n"); + print_hex_dump(KERN_DEBUG, "aaudio:OUT ", DUMP_PREFIX_NONE, 32, 1, ctx->msg.data, ctx->msg.size, true); +#endif + bce_set_submission_single(s, b->qout.dma_addr + (dma_addr_t) (ctx->msg.data - b->qout.data), ctx->msg.size); + bce_submit_to_device(b->qout.sq); + b->qout.data_tail = (b->qout.data_tail + 1) % b->qout.el_count; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); +} + +int __aaudio_send_cmd_sync(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, struct aaudio_msg *reply) +{ + struct aaudio_bce_queue_entry ent; + DECLARE_COMPLETION_ONSTACK(cmpl); + ent.msg = reply; + ent.cmpl = &cmpl; + b->pending_entries[ctx->tag_n] = &ent; + __aaudio_send(b, ctx); /* unlocks the spinlock */ + ctx->timeout = wait_for_completion_timeout(&cmpl, ctx->timeout); + if (ctx->timeout == 0) { + /* Remove the pending queue entry; this will be normally handled by the completion route but + * during a timeout it won't */ + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + if (b->pending_entries[ctx->tag_n] == &ent) + b->pending_entries[ctx->tag_n] = NULL; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); + return -ETIMEDOUT; + } + return 0; +} + +static void aaudio_handle_reply(struct aaudio_bce *b, struct aaudio_msg *reply) +{ + const char *tag; + int tagn; + unsigned long irq_flags; + char tag_zero[5]; + struct aaudio_bce_queue_entry *entry; + + tag = ((struct aaudio_msg_header *) reply->data)->tag; + if (tag[0] != 'S') { + pr_err("aaudio_handle_reply: Unexpected tag: %.4s\n", tag); + return; + } + *((u32 *) tag_zero) = *((u32 *) tag); + tag_zero[4] = 0; + if (kstrtoint(&tag_zero[1], 10, &tagn)) { + pr_err("aaudio_handle_reply: Tag parse failed: %.4s\n", tag); + return; + } + + spin_lock_irqsave(&b->spinlock, irq_flags); + entry = b->pending_entries[tagn]; + if (entry) { + if (reply->size < entry->msg->size) + entry->msg->size = reply->size; + memcpy(entry->msg->data, reply->data, entry->msg->size); + complete(entry->cmpl); + + b->pending_entries[tagn] = NULL; + } else { + pr_err("aaudio_handle_reply: No queued item found for tag: %.4s\n", tag); + } + spin_unlock_irqrestore(&b->spinlock, irq_flags); +} + +static void aaudio_bce_out_queue_completion(struct bce_queue_sq *sq) +{ + while (bce_next_completion(sq)) { + //pr_info("aaudio: Send confirmed\n"); + bce_notify_submission_complete(sq); + } +} + +static void aaudio_bce_in_queue_handle_msg(struct aaudio_device *a, struct aaudio_msg *msg); + +static void aaudio_bce_in_queue_completion(struct bce_queue_sq *sq) +{ + struct aaudio_msg msg; + struct aaudio_device *dev = sq->userdata; + struct aaudio_bce_queue *q = &dev->bcem.qin; + struct bce_sq_completion_data *c; + size_t cnt = 0; + + mb(); + while ((c = bce_next_completion(sq))) { + msg.data = (u8 *) q->data + q->data_head * q->el_size; + msg.size = c->data_size; +#ifdef DEBUG + pr_debug("aaudio: Received command data %llx\n", c->data_size); + print_hex_dump(KERN_DEBUG, "aaudio:IN ", DUMP_PREFIX_NONE, 32, 1, msg.data, min(msg.size, 128UL), true); +#endif + aaudio_bce_in_queue_handle_msg(dev, &msg); + + q->data_head = (q->data_head + 1) % q->el_count; + + bce_notify_submission_complete(sq); + ++cnt; + } + aaudio_bce_in_queue_submit_pending(q, cnt); +} + +static void aaudio_bce_in_queue_handle_msg(struct aaudio_device *a, struct aaudio_msg *msg) +{ + struct aaudio_msg_header *header = (struct aaudio_msg_header *) msg->data; + if (msg->size < sizeof(struct aaudio_msg_header)) { + pr_err("aaudio: Msg size smaller than header (%lx)", msg->size); + return; + } + if (header->type == AAUDIO_MSG_TYPE_RESPONSE) { + aaudio_handle_reply(&a->bcem, msg); + } else if (header->type == AAUDIO_MSG_TYPE_COMMAND) { + aaudio_handle_command(a, msg); + } else if (header->type == AAUDIO_MSG_TYPE_NOTIFICATION) { + aaudio_handle_notification(a, msg); + } +} + +void aaudio_bce_in_queue_submit_pending(struct aaudio_bce_queue *q, size_t count) +{ + struct bce_qe_submission *s; + while (count--) { + if (bce_reserve_submission(q->sq, NULL)) { + pr_err("aaudio: Failed to reserve an event queue submission\n"); + break; + } + s = bce_next_submission(q->sq); + bce_set_submission_single(s, q->dma_addr + (dma_addr_t) (q->data_tail * q->el_size), q->el_size); + q->data_tail = (q->data_tail + 1) % q->el_count; + } + bce_submit_to_device(q->sq); +} + +struct aaudio_msg aaudio_reply_alloc(void) +{ + struct aaudio_msg ret; + ret.size = AAUDIO_BCE_QUEUE_ELEMENT_SIZE; + ret.data = kmalloc(ret.size, GFP_KERNEL); + return ret; +} + +void aaudio_reply_free(struct aaudio_msg *reply) +{ + kfree(reply->data); +} diff --git a/drivers/staging/apple-bce/audio/protocol_bce.h b/drivers/staging/apple-bce/audio/protocol_bce.h new file mode 100644 index 000000000000..14d26c05ddf9 --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol_bce.h @@ -0,0 +1,72 @@ +#ifndef AAUDIO_PROTOCOL_BCE_H +#define AAUDIO_PROTOCOL_BCE_H + +#include "protocol.h" +#include "../queue.h" + +#define AAUDIO_BCE_QUEUE_ELEMENT_SIZE 0x1000 +#define AAUDIO_BCE_QUEUE_ELEMENT_COUNT 20 + +#define AAUDIO_BCE_QUEUE_TAG_COUNT 1000 + +struct aaudio_device; + +struct aaudio_bce_queue_entry { + struct aaudio_msg *msg; + struct completion *cmpl; +}; +struct aaudio_bce_queue { + struct bce_queue_cq *cq; + struct bce_queue_sq *sq; + void *data; + dma_addr_t dma_addr; + size_t data_head, data_tail; + size_t el_size, el_count; +}; +struct aaudio_bce { + struct bce_queue_cq *cq; + struct aaudio_bce_queue qin; + struct aaudio_bce_queue qout; + int tag_num; + struct aaudio_bce_queue_entry *pending_entries[AAUDIO_BCE_QUEUE_TAG_COUNT]; + struct spinlock spinlock; +}; + +struct aaudio_send_ctx { + int status; + int tag_n; + unsigned long irq_flags; + struct aaudio_msg msg; + unsigned long timeout; +}; + +int aaudio_bce_init(struct aaudio_device *dev); +int __aaudio_send_prepare(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, char *tag); +void __aaudio_send(struct aaudio_bce *b, struct aaudio_send_ctx *ctx); +int __aaudio_send_cmd_sync(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, struct aaudio_msg *reply); + +#define aaudio_send_with_tag(a, ctx, tag, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __aaudio_send_prepare(&(a)->bcem, (ctx), (tag)); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + __aaudio_send(&(a)->bcem, (ctx)); \ + } \ + (ctx)->status; \ +}) +#define aaudio_send(a, ctx, tout, fn, ...) aaudio_send_with_tag(a, ctx, NULL, tout, fn, ##__VA_ARGS__) + +#define aaudio_send_cmd_sync(a, ctx, reply, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __aaudio_send_prepare(&(a)->bcem, (ctx), NULL); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + (ctx)->status = __aaudio_send_cmd_sync(&(a)->bcem, (ctx), (reply)); \ + } \ + (ctx)->status; \ +}) + +struct aaudio_msg aaudio_reply_alloc(void); +void aaudio_reply_free(struct aaudio_msg *reply); + +#endif //AAUDIO_PROTOCOL_BCE_H diff --git a/drivers/staging/apple-bce/mailbox.c b/drivers/staging/apple-bce/mailbox.c new file mode 100644 index 000000000000..07ce11928988 --- /dev/null +++ b/drivers/staging/apple-bce/mailbox.c @@ -0,0 +1,155 @@ +#include "mailbox.h" +#include +#include "apple_bce.h" +#include + +#define REG_MBOX_OUT_BASE 0x820 +#define REG_MBOX_REPLY_COUNTER 0x108 +#define REG_MBOX_REPLY_BASE 0x810 +#define REG_TIMESTAMP_BASE 0xC000 + +#define BCE_MBOX_TIMEOUT_MS 200 + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb) +{ + mb->reg_mb = reg_mb; + init_completion(&mb->mb_completion); +} + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv) +{ + u32 __iomem *regb; + + if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { + return -EEXIST; // We don't support two messages at once + } + reinit_completion(&mb->mb_completion); + + pr_debug("bce_mailbox_send: %llx\n", msg); + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); + iowrite32((u32) msg, regb); + iowrite32((u32) (msg >> 32), regb + 1); + iowrite32(0, regb + 2); + iowrite32(0, regb + 3); + + wait_for_completion_timeout(&mb->mb_completion, msecs_to_jiffies(BCE_MBOX_TIMEOUT_MS)); + if (atomic_read(&mb->mb_status) != 2) { // Didn't get the reply + atomic_set(&mb->mb_status, 0); + return -ETIMEDOUT; + } + + *recv = mb->mb_result; + pr_debug("bce_mailbox_send: reply %llx\n", *recv); + + atomic_set(&mb->mb_status, 0); + return 0; +} + +static int bce_mailbox_retrive_response(struct bce_mailbox *mb) +{ + u32 __iomem *regb; + u32 lo, hi; + int count, counter; + u32 res = ioread32((u8*) mb->reg_mb + REG_MBOX_REPLY_COUNTER); + count = (res >> 20) & 0xf; + counter = count; + pr_debug("bce_mailbox_retrive_response count=%i\n", count); + while (counter--) { + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_REPLY_BASE); + lo = ioread32(regb); + hi = ioread32(regb + 1); + ioread32(regb + 2); + ioread32(regb + 3); + pr_debug("bce_mailbox_retrive_response %llx\n", ((u64) hi << 32) | lo); + mb->mb_result = ((u64) hi << 32) | lo; + } + return count > 0 ? 0 : -ENODATA; +} + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb) +{ + int status = bce_mailbox_retrive_response(mb); + if (!status) { + atomic_set(&mb->mb_status, 2); + complete(&mb->mb_completion); + } + return status; +} + +static void bc_send_timestamp(struct timer_list *tl); + +void bce_timestamp_init(struct bce_timestamp *ts, void __iomem *reg) +{ + u32 __iomem *regb; + + spin_lock_init(&ts->stop_sl); + ts->stopped = false; + + ts->reg = reg; + + regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + + ioread32(regb); + mb(); + + timer_setup(&ts->timer, bc_send_timestamp, 0); +} + +void bce_timestamp_start(struct bce_timestamp *ts, bool is_initial) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + + if (is_initial) { + iowrite32((u32) -4, regb + 2); + iowrite32((u32) -1, regb); + } else { + iowrite32((u32) -3, regb + 2); + iowrite32((u32) -1, regb); + } + + spin_lock_irqsave(&ts->stop_sl, flags); + ts->stopped = false; + spin_unlock_irqrestore(&ts->stop_sl, flags); + mod_timer(&ts->timer, jiffies + msecs_to_jiffies(150)); +} + +void bce_timestamp_stop(struct bce_timestamp *ts) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + + spin_lock_irqsave(&ts->stop_sl, flags); + ts->stopped = true; + spin_unlock_irqrestore(&ts->stop_sl, flags); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,15,0) + del_timer_sync(&ts->timer); +#else + timer_delete_sync(&ts->timer); +#endif + iowrite32((u32) -2, regb + 2); + iowrite32((u32) -1, regb); +} + +static void bc_send_timestamp(struct timer_list *tl) +{ + struct bce_timestamp *ts; + unsigned long flags; + u32 __iomem *regb; + ktime_t bt; + + ts = container_of(tl, struct bce_timestamp, timer); + regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + local_irq_save(flags); + ioread32(regb + 2); + mb(); + bt = ktime_get_boottime(); + iowrite32((u32) bt, regb + 2); + iowrite32((u32) (bt >> 32), regb); + + spin_lock(&ts->stop_sl); + if (!ts->stopped) + mod_timer(&ts->timer, jiffies + msecs_to_jiffies(150)); + spin_unlock(&ts->stop_sl); + local_irq_restore(flags); +} diff --git a/drivers/staging/apple-bce/mailbox.h b/drivers/staging/apple-bce/mailbox.h new file mode 100644 index 000000000000..f3323f95ba51 --- /dev/null +++ b/drivers/staging/apple-bce/mailbox.h @@ -0,0 +1,53 @@ +#ifndef BCE_MAILBOX_H +#define BCE_MAILBOX_H + +#include +#include +#include + +struct bce_mailbox { + void __iomem *reg_mb; + + atomic_t mb_status; // possible statuses: 0 (no msg), 1 (has active msg), 2 (got reply) + struct completion mb_completion; + uint64_t mb_result; +}; + +enum bce_message_type { + BCE_MB_REGISTER_COMMAND_SQ = 0x7, // to-device + BCE_MB_REGISTER_COMMAND_CQ = 0x8, // to-device + BCE_MB_REGISTER_COMMAND_QUEUE_REPLY = 0xB, // to-host + BCE_MB_SET_FW_PROTOCOL_VERSION = 0xC, // both + BCE_MB_SLEEP_NO_STATE = 0x14, // to-device + BCE_MB_RESTORE_NO_STATE = 0x15, // to-device + BCE_MB_SAVE_STATE_AND_SLEEP = 0x17, // to-device + BCE_MB_RESTORE_STATE_AND_WAKE = 0x18, // to-device + BCE_MB_SAVE_STATE_AND_SLEEP_FAILURE = 0x19, // from-device + BCE_MB_SAVE_RESTORE_STATE_COMPLETE = 0x1A, // from-device +}; + +#define BCE_MB_MSG(type, value) (((u64) (type) << 58) | ((value) & 0x3FFFFFFFFFFFFFFLL)) +#define BCE_MB_TYPE(v) ((u32) (v >> 58)) +#define BCE_MB_VALUE(v) (v & 0x3FFFFFFFFFFFFFFLL) + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb); + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv); + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb); + + +struct bce_timestamp { + void __iomem *reg; + struct timer_list timer; + struct spinlock stop_sl; + bool stopped; +}; + +void bce_timestamp_init(struct bce_timestamp *ts, void __iomem *reg); + +void bce_timestamp_start(struct bce_timestamp *ts, bool is_initial); + +void bce_timestamp_stop(struct bce_timestamp *ts); + +#endif //BCEDRIVER_MAILBOX_H diff --git a/drivers/staging/apple-bce/queue.c b/drivers/staging/apple-bce/queue.c new file mode 100644 index 000000000000..bc9cd3bc6f0c --- /dev/null +++ b/drivers/staging/apple-bce/queue.c @@ -0,0 +1,390 @@ +#include "queue.h" +#include "apple_bce.h" + +#define REG_DOORBELL_BASE 0x44000 + +struct bce_queue_cq *bce_alloc_cq(struct apple_bce_device *dev, int qid, u32 el_count) +{ + struct bce_queue_cq *q; + q = kzalloc(sizeof(struct bce_queue_cq), GFP_KERNEL); + q->qid = qid; + q->type = BCE_QUEUE_CQ; + q->el_count = el_count; + q->data = dma_alloc_coherent(&dev->pci->dev, el_count * sizeof(struct bce_qe_completion), + &q->dma_handle, GFP_KERNEL); + if (!q->data) { + pr_err("DMA queue memory alloc failed\n"); + kfree(q); + return NULL; + } + return q; +} + +void bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) cq->qid; + cfg->el_count = (u16) cq->el_count; + cfg->vector_or_cq = 0; + cfg->_pad = 0; + cfg->addr = cq->dma_handle; + cfg->length = cq->el_count * sizeof(struct bce_qe_completion); +} + +void bce_free_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq) +{ + dma_free_coherent(&dev->pci->dev, cq->el_count * sizeof(struct bce_qe_completion), cq->data, cq->dma_handle); + kfree(cq); +} + +static void bce_handle_cq_completion(struct apple_bce_device *dev, struct bce_qe_completion *e, size_t *ce) +{ + struct bce_queue *target; + struct bce_queue_sq *target_sq; + struct bce_sq_completion_data *cmpl; + if (e->qid >= BCE_MAX_QUEUE_COUNT) { + pr_err("Device sent a response for qid (%u) >= BCE_MAX_QUEUE_COUNT\n", e->qid); + return; + } + target = dev->queues[e->qid]; + if (!target || target->type != BCE_QUEUE_SQ) { + pr_err("Device sent a response for qid (%u), which does not exist\n", e->qid); + return; + } + target_sq = (struct bce_queue_sq *) target; + if (target_sq->completion_tail != e->completion_index) { + pr_err("Completion index mismatch; this is likely going to make this driver unusable\n"); + return; + } + if (!target_sq->has_pending_completions) { + target_sq->has_pending_completions = true; + dev->int_sq_list[(*ce)++] = target_sq; + } + cmpl = &target_sq->completion_data[e->completion_index]; + cmpl->status = e->status; + cmpl->data_size = e->data_size; + cmpl->result = e->result; + wmb(); + target_sq->completion_tail = (target_sq->completion_tail + 1) % target_sq->el_count; +} + +void bce_handle_cq_completions(struct apple_bce_device *dev, struct bce_queue_cq *cq) +{ + size_t ce = 0; + struct bce_qe_completion *e; + struct bce_queue_sq *sq; + e = bce_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + return; + mb(); + while (true) { + e = bce_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + break; + // pr_info("apple-bce: compl: %i: %i %llx %llx", e->qid, e->status, e->data_size, e->result); + bce_handle_cq_completion(dev, e, &ce); + e->flags = 0; + cq->index = (cq->index + 1) % cq->el_count; + } + mb(); + iowrite32(cq->index, (u32 *) ((u8 *) dev->reg_mem_dma + REG_DOORBELL_BASE) + cq->qid); + while (ce) { + --ce; + sq = dev->int_sq_list[ce]; + sq->completion(sq); + sq->has_pending_completions = false; + } +} + + +struct bce_queue_sq *bce_alloc_sq(struct apple_bce_device *dev, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *q; + q = kzalloc(sizeof(struct bce_queue_sq), GFP_KERNEL); + q->qid = qid; + q->type = BCE_QUEUE_SQ; + q->el_size = el_size; + q->el_count = el_count; + q->data = dma_alloc_coherent(&dev->pci->dev, el_count * el_size, + &q->dma_handle, GFP_KERNEL); + q->completion = compl; + q->userdata = userdata; + q->completion_data = kzalloc(sizeof(struct bce_sq_completion_data) * el_count, GFP_KERNEL); + q->reg_mem_dma = dev->reg_mem_dma; + atomic_set(&q->available_commands, el_count - 1); + init_completion(&q->available_command_completion); + atomic_set(&q->available_command_completion_waiting_count, 0); + if (!q->data) { + pr_err("DMA queue memory alloc failed\n"); + kfree(q); + return NULL; + } + return q; +} + +void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) sq->qid; + cfg->el_count = (u16) sq->el_count; + cfg->vector_or_cq = (u16) cq->qid; + cfg->_pad = 0; + cfg->addr = sq->dma_handle; + cfg->length = sq->el_count * sq->el_size; +} + +void bce_free_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq) +{ + dma_free_coherent(&dev->pci->dev, sq->el_count * sq->el_size, sq->data, sq->dma_handle); + kfree(sq); +} + +int bce_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout) +{ + while (atomic_dec_if_positive(&sq->available_commands) < 0) { + if (!timeout || !*timeout) + return -EAGAIN; + atomic_inc(&sq->available_command_completion_waiting_count); + *timeout = wait_for_completion_timeout(&sq->available_command_completion, *timeout); + if (!*timeout) { + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) < 0) + try_wait_for_completion(&sq->available_command_completion); /* consume the pending completion */ + } + } + return 0; +} + +void bce_cancel_submission_reservation(struct bce_queue_sq *sq) +{ + atomic_inc(&sq->available_commands); +} + +void *bce_next_submission(struct bce_queue_sq *sq) +{ + void *ret = bce_sq_element(sq, sq->tail); + sq->tail = (sq->tail + 1) % sq->el_count; + return ret; +} + +void bce_submit_to_device(struct bce_queue_sq *sq) +{ + mb(); + iowrite32(sq->tail, (u32 *) ((u8 *) sq->reg_mem_dma + REG_DOORBELL_BASE) + sq->qid); +} + +void bce_notify_submission_complete(struct bce_queue_sq *sq) +{ + sq->head = (sq->head + 1) % sq->el_count; + atomic_inc(&sq->available_commands); + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) >= 0) { + complete(&sq->available_command_completion); + } +} + +void bce_set_submission_single(struct bce_qe_submission *element, dma_addr_t addr, size_t size) +{ + element->addr = addr; + element->length = size; + element->segl_addr = element->segl_length = 0; +} + +static void bce_cmdq_completion(struct bce_queue_sq *q); + +struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_device *dev, int qid, u32 el_count) +{ + struct bce_queue_cmdq *q; + q = kzalloc(sizeof(struct bce_queue_cmdq), GFP_KERNEL); + q->sq = bce_alloc_sq(dev, qid, BCE_CMD_SIZE, el_count, bce_cmdq_completion, q); + if (!q->sq) { + kfree(q); + return NULL; + } + spin_lock_init(&q->lck); + q->tres = kzalloc(sizeof(struct bce_queue_cmdq_result_el*) * el_count, GFP_KERNEL); + if (!q->tres) { + kfree(q); + return NULL; + } + return q; +} + +void bce_free_cmdq(struct apple_bce_device *dev, struct bce_queue_cmdq *cmdq) +{ + bce_free_sq(dev, cmdq->sq); + kfree(cmdq->tres); + kfree(cmdq); +} + +void bce_cmdq_completion(struct bce_queue_sq *q) +{ + struct bce_queue_cmdq_result_el *el; + struct bce_queue_cmdq *cmdq = q->userdata; + struct bce_sq_completion_data *result; + + spin_lock(&cmdq->lck); + while ((result = bce_next_completion(q))) { + el = cmdq->tres[cmdq->sq->head]; + if (el) { + el->result = result->result; + el->status = result->status; + mb(); + complete(&el->cmpl); + } else { + pr_err("apple-bce: Unexpected command queue completion\n"); + } + cmdq->tres[cmdq->sq->head] = NULL; + bce_notify_submission_complete(q); + } + spin_unlock(&cmdq->lck); +} + +static __always_inline void *bce_cmd_start(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + void *ret; + unsigned long timeout; + init_completion(&res->cmpl); + mb(); + + timeout = msecs_to_jiffies(1000L * 60 * 5); /* wait for up to ~5 minutes */ + if (bce_reserve_submission(cmdq->sq, &timeout)) + return NULL; + + spin_lock(&cmdq->lck); + cmdq->tres[cmdq->sq->tail] = res; + ret = bce_next_submission(cmdq->sq); + return ret; +} + +static __always_inline void bce_cmd_finish(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + bce_submit_to_device(cmdq->sq); + spin_unlock(&cmdq->lck); + + wait_for_completion(&res->cmpl); + mb(); +} + +u32 bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_register_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_REGISTER_MEMORY_QUEUE; + cmd->flags = (u16) ((name ? 2 : 0) | (isdirout ? 1 : 0)); + cmd->qid = cfg->qid; + cmd->el_count = cfg->el_count; + cmd->vector_or_cq = cfg->vector_or_cq; + memset(cmd->name, 0, sizeof(cmd->name)); + if (name) { + cmd->name_len = (u16) min(strlen(name), (size_t) sizeof(cmd->name)); + memcpy(cmd->name, name, cmd->name_len); + } else { + cmd->name_len = 0; + } + cmd->addr = cfg->addr; + cmd->length = cfg->length; + + bce_cmd_finish(cmdq, &res); + return res.status; +} + +u32 bce_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_UNREGISTER_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + bce_cmd_finish(cmdq, &res); + return res.status; +} + +u32 bce_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_FLUSH_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + bce_cmd_finish(cmdq, &res); + return res.status; +} + + +struct bce_queue_cq *bce_create_cq(struct apple_bce_device *dev, u32 el_count) +{ + struct bce_queue_cq *cq; + struct bce_queue_memcfg cfg; + int qid = ida_simple_get(&dev->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); + if (qid < 0) + return NULL; + cq = bce_alloc_cq(dev, qid, el_count); + if (!cq) + return NULL; + bce_get_cq_memcfg(cq, &cfg); + if (bce_cmd_register_queue(dev->cmd_cmdq, &cfg, NULL, false) != 0) { + pr_err("apple-bce: CQ registration failed (%i)", qid); + bce_free_cq(dev, cq); + ida_simple_remove(&dev->queue_ida, (uint) qid); + return NULL; + } + dev->queues[qid] = (struct bce_queue *) cq; + return cq; +} + +struct bce_queue_sq *bce_create_sq(struct apple_bce_device *dev, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *sq; + struct bce_queue_memcfg cfg; + int qid; + if (cq == NULL) + return NULL; /* cq can not be null */ + if (name == NULL) + return NULL; /* name can not be null */ + if (direction != DMA_TO_DEVICE && direction != DMA_FROM_DEVICE) + return NULL; /* unsupported direction */ + qid = ida_simple_get(&dev->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); + if (qid < 0) + return NULL; + sq = bce_alloc_sq(dev, qid, sizeof(struct bce_qe_submission), el_count, compl, userdata); + if (!sq) + return NULL; + bce_get_sq_memcfg(sq, cq, &cfg); + if (bce_cmd_register_queue(dev->cmd_cmdq, &cfg, name, direction != DMA_FROM_DEVICE) != 0) { + pr_err("apple-bce: SQ registration failed (%i)", qid); + bce_free_sq(dev, sq); + ida_simple_remove(&dev->queue_ida, (uint) qid); + return NULL; + } + spin_lock(&dev->queues_lock); + dev->queues[qid] = (struct bce_queue *) sq; + spin_unlock(&dev->queues_lock); + return sq; +} + +void bce_destroy_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq) +{ + if (!dev->is_being_removed && bce_cmd_unregister_memory_queue(dev->cmd_cmdq, (u16) cq->qid)) + pr_err("apple-bce: CQ unregister failed"); + spin_lock(&dev->queues_lock); + dev->queues[cq->qid] = NULL; + spin_unlock(&dev->queues_lock); + ida_simple_remove(&dev->queue_ida, (uint) cq->qid); + bce_free_cq(dev, cq); +} + +void bce_destroy_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq) +{ + if (!dev->is_being_removed && bce_cmd_unregister_memory_queue(dev->cmd_cmdq, (u16) sq->qid)) + pr_err("apple-bce: CQ unregister failed"); + spin_lock(&dev->queues_lock); + dev->queues[sq->qid] = NULL; + spin_unlock(&dev->queues_lock); + ida_simple_remove(&dev->queue_ida, (uint) sq->qid); + bce_free_sq(dev, sq); +} \ No newline at end of file diff --git a/drivers/staging/apple-bce/queue.h b/drivers/staging/apple-bce/queue.h new file mode 100644 index 000000000000..8368ac5dfca8 --- /dev/null +++ b/drivers/staging/apple-bce/queue.h @@ -0,0 +1,177 @@ +#ifndef BCE_QUEUE_H +#define BCE_QUEUE_H + +#include +#include + +#define BCE_CMD_SIZE 0x40 + +struct apple_bce_device; + +enum bce_queue_type { + BCE_QUEUE_CQ, BCE_QUEUE_SQ +}; +struct bce_queue { + int qid; + int type; +}; +struct bce_queue_cq { + int qid; + int type; + u32 el_count; + dma_addr_t dma_handle; + void *data; + + u32 index; +}; +struct bce_queue_sq; +typedef void (*bce_sq_completion)(struct bce_queue_sq *q); +struct bce_sq_completion_data { + u32 status; + u64 data_size; + u64 result; +}; +struct bce_queue_sq { + int qid; + int type; + u32 el_size; + u32 el_count; + dma_addr_t dma_handle; + void *data; + void *userdata; + void __iomem *reg_mem_dma; + + atomic_t available_commands; + struct completion available_command_completion; + atomic_t available_command_completion_waiting_count; + u32 head, tail; + + u32 completion_cidx, completion_tail; + struct bce_sq_completion_data *completion_data; + bool has_pending_completions; + bce_sq_completion completion; +}; + +struct bce_queue_cmdq_result_el { + struct completion cmpl; + u32 status; + u64 result; +}; +struct bce_queue_cmdq { + struct bce_queue_sq *sq; + struct spinlock lck; + struct bce_queue_cmdq_result_el **tres; +}; + +struct bce_queue_memcfg { + u16 qid; + u16 el_count; + u16 vector_or_cq; + u16 _pad; + u64 addr; + u64 length; +}; + +enum bce_qe_completion_status { + BCE_COMPLETION_SUCCESS = 0, + BCE_COMPLETION_ERROR = 1, + BCE_COMPLETION_ABORTED = 2, + BCE_COMPLETION_NO_SPACE = 3, + BCE_COMPLETION_OVERRUN = 4 +}; +enum bce_qe_completion_flags { + BCE_COMPLETION_FLAG_PENDING = 0x8000 +}; +struct bce_qe_completion { + u64 result; + u64 data_size; + u16 qid; + u16 completion_index; + u16 status; // bce_qe_completion_status + u16 flags; // bce_qe_completion_flags +}; + +struct bce_qe_submission { + u64 length; + u64 addr; + + u64 segl_addr; + u64 segl_length; +}; + +enum bce_cmdq_command { + BCE_CMD_REGISTER_MEMORY_QUEUE = 0x20, + BCE_CMD_UNREGISTER_MEMORY_QUEUE = 0x30, + BCE_CMD_FLUSH_MEMORY_QUEUE = 0x40, + BCE_CMD_SET_MEMORY_QUEUE_PROPERTY = 0x50 +}; +struct bce_cmdq_simple_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; +}; +struct bce_cmdq_register_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; + u16 _pad; + u16 el_count; + u16 vector_or_cq; + u16 _pad2; + u16 name_len; + char name[0x20]; + u64 addr; + u64 length; +}; + +static __always_inline void *bce_sq_element(struct bce_queue_sq *q, int i) { + return (void *) ((u8 *) q->data + q->el_size * i); +} +static __always_inline void *bce_cq_element(struct bce_queue_cq *q, int i) { + return (void *) ((struct bce_qe_completion *) q->data + i); +} + +static __always_inline struct bce_sq_completion_data *bce_next_completion(struct bce_queue_sq *sq) { + struct bce_sq_completion_data *res; + rmb(); + if (sq->completion_cidx == sq->completion_tail) + return NULL; + res = &sq->completion_data[sq->completion_cidx]; + sq->completion_cidx = (sq->completion_cidx + 1) % sq->el_count; + return res; +} + +struct bce_queue_cq *bce_alloc_cq(struct apple_bce_device *dev, int qid, u32 el_count); +void bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void bce_free_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq); +void bce_handle_cq_completions(struct apple_bce_device *dev, struct bce_queue_cq *cq); + +struct bce_queue_sq *bce_alloc_sq(struct apple_bce_device *dev, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata); +void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void bce_free_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq); +int bce_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout); +void bce_cancel_submission_reservation(struct bce_queue_sq *sq); +void *bce_next_submission(struct bce_queue_sq *sq); +void bce_submit_to_device(struct bce_queue_sq *sq); +void bce_notify_submission_complete(struct bce_queue_sq *sq); + +void bce_set_submission_single(struct bce_qe_submission *element, dma_addr_t addr, size_t size); + +struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_device *dev, int qid, u32 el_count); +void bce_free_cmdq(struct apple_bce_device *dev, struct bce_queue_cmdq *cmdq); + +u32 bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout); +u32 bce_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); +u32 bce_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); + + +/* User API - Creates and registers the queue */ + +struct bce_queue_cq *bce_create_cq(struct apple_bce_device *dev, u32 el_count); +struct bce_queue_sq *bce_create_sq(struct apple_bce_device *dev, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata); +void bce_destroy_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq); +void bce_destroy_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq); + +#endif //BCEDRIVER_MAILBOX_H diff --git a/drivers/staging/apple-bce/queue_dma.c b/drivers/staging/apple-bce/queue_dma.c new file mode 100644 index 000000000000..b236613285c0 --- /dev/null +++ b/drivers/staging/apple-bce/queue_dma.c @@ -0,0 +1,220 @@ +#include "queue_dma.h" +#include +#include +#include "queue.h" + +static int bce_alloc_scatterlist_from_vm(struct sg_table *tbl, void *data, size_t len); +static struct bce_segment_list_element_hostinfo *bce_map_segment_list( + struct device *dev, struct scatterlist *pages, int pagen); +static void bce_unmap_segement_list(struct device *dev, struct bce_segment_list_element_hostinfo *list); + +int bce_map_dma_buffer(struct device *dev, struct bce_dma_buffer *buf, struct sg_table scatterlist, + enum dma_data_direction dir) +{ + int cnt; + + buf->direction = dir; + buf->scatterlist = scatterlist; + buf->seglist_hostinfo = NULL; + + cnt = dma_map_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); + if (cnt != buf->scatterlist.nents) { + pr_err("apple-bce: DMA scatter list mapping returned an unexpected count: %i\n", cnt); + dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); + return -EIO; + } + if (cnt == 1) + return 0; + + buf->seglist_hostinfo = bce_map_segment_list(dev, buf->scatterlist.sgl, buf->scatterlist.nents); + if (!buf->seglist_hostinfo) { + pr_err("apple-bce: Creating segment list failed\n"); + dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); + return -EIO; + } + return 0; +} + +int bce_map_dma_buffer_vm(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir) +{ + int status; + struct sg_table scatterlist; + if ((status = bce_alloc_scatterlist_from_vm(&scatterlist, data, len))) + return status; + if ((status = bce_map_dma_buffer(dev, buf, scatterlist, dir))) { + sg_free_table(&scatterlist); + return status; + } + return 0; +} + +int bce_map_dma_buffer_km(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir) +{ + /* Kernel memory is continuous which is great for us. */ + int status; + struct sg_table scatterlist; + if ((status = sg_alloc_table(&scatterlist, 1, GFP_KERNEL))) { + sg_free_table(&scatterlist); + return status; + } + sg_set_buf(scatterlist.sgl, data, (uint) len); + if ((status = bce_map_dma_buffer(dev, buf, scatterlist, dir))) { + sg_free_table(&scatterlist); + return status; + } + return 0; +} + +void bce_unmap_dma_buffer(struct device *dev, struct bce_dma_buffer *buf) +{ + dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, buf->direction); + bce_unmap_segement_list(dev, buf->seglist_hostinfo); +} + + +static int bce_alloc_scatterlist_from_vm(struct sg_table *tbl, void *data, size_t len) +{ + int status, i; + struct page **pages; + size_t off, start_page, end_page, page_count; + off = (size_t) data % PAGE_SIZE; + start_page = (size_t) data / PAGE_SIZE; + end_page = ((size_t) data + len - 1) / PAGE_SIZE; + page_count = end_page - start_page + 1; + + if (page_count > PAGE_SIZE / sizeof(struct page *)) + pages = vmalloc(page_count * sizeof(struct page *)); + else + pages = kmalloc(page_count * sizeof(struct page *), GFP_KERNEL); + + for (i = 0; i < page_count; i++) + pages[i] = vmalloc_to_page((void *) ((start_page + i) * PAGE_SIZE)); + + if ((status = sg_alloc_table_from_pages(tbl, pages, page_count, (unsigned int) off, len, GFP_KERNEL))) { + sg_free_table(tbl); + } + + if (page_count > PAGE_SIZE / sizeof(struct page *)) + vfree(pages); + else + kfree(pages); + return status; +} + +#define BCE_ELEMENTS_PER_PAGE ((PAGE_SIZE - sizeof(struct bce_segment_list_header)) \ + / sizeof(struct bce_segment_list_element)) +#define BCE_ELEMENTS_PER_ADDITIONAL_PAGE (PAGE_SIZE / sizeof(struct bce_segment_list_element)) + +static struct bce_segment_list_element_hostinfo *bce_map_segment_list( + struct device *dev, struct scatterlist *pages, int pagen) +{ + size_t ptr, pptr = 0; + struct bce_segment_list_header theader; /* a temp header, to store the initial seg */ + struct bce_segment_list_header *header; + struct bce_segment_list_element *el, *el_end; + struct bce_segment_list_element_hostinfo *out, *pout, *out_root; + struct scatterlist *sg; + int i; + header = &theader; + out = out_root = NULL; + el = el_end = NULL; + for_each_sg(pages, sg, pagen, i) { + if (el >= el_end) { + /* allocate a new page, this will be also done for the first element */ + ptr = __get_free_page(GFP_KERNEL); + if (pptr && ptr == pptr + PAGE_SIZE) { + out->page_count++; + header->element_count += BCE_ELEMENTS_PER_ADDITIONAL_PAGE; + el_end += BCE_ELEMENTS_PER_ADDITIONAL_PAGE; + } else { + header = (void *) ptr; + header->element_count = BCE_ELEMENTS_PER_PAGE; + header->data_size = 0; + header->next_segl_addr = 0; + header->next_segl_length = 0; + el = (void *) (header + 1); + el_end = el + BCE_ELEMENTS_PER_PAGE; + + if (out) { + out->next = kmalloc(sizeof(struct bce_segment_list_element_hostinfo), GFP_KERNEL); + out = out->next; + } else { + out_root = out = kmalloc(sizeof(struct bce_segment_list_element_hostinfo), GFP_KERNEL); + } + out->page_start = (void *) ptr; + out->page_count = 1; + out->dma_start = DMA_MAPPING_ERROR; + out->next = NULL; + } + pptr = ptr; + } + el->addr = sg->dma_address; + el->length = sg->length; + header->data_size += el->length; + } + + /* DMA map */ + out = out_root; + pout = NULL; + while (out) { + out->dma_start = dma_map_single(dev, out->page_start, out->page_count * PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, out->dma_start)) + goto error; + if (pout) { + header = pout->page_start; + header->next_segl_addr = out->dma_start; + header->next_segl_length = out->page_count * PAGE_SIZE; + } + pout = out; + out = out->next; + } + return out_root; + + error: + bce_unmap_segement_list(dev, out_root); + return NULL; +} + +static void bce_unmap_segement_list(struct device *dev, struct bce_segment_list_element_hostinfo *list) +{ + struct bce_segment_list_element_hostinfo *next; + while (list) { + if (list->dma_start != DMA_MAPPING_ERROR) + dma_unmap_single(dev, list->dma_start, list->page_count * PAGE_SIZE, DMA_TO_DEVICE); + next = list->next; + kfree(list); + list = next; + } +} + +int bce_set_submission_buf(struct bce_qe_submission *element, struct bce_dma_buffer *buf, size_t offset, size_t length) +{ + struct bce_segment_list_element_hostinfo *seg; + struct bce_segment_list_header *seg_header; + + seg = buf->seglist_hostinfo; + if (!seg) { + element->addr = buf->scatterlist.sgl->dma_address + offset; + element->length = length; + element->segl_addr = 0; + element->segl_length = 0; + return 0; + } + + while (seg) { + seg_header = seg->page_start; + if (offset <= seg_header->data_size) + break; + offset -= seg_header->data_size; + seg = seg->next; + } + if (!seg) + return -EINVAL; + element->addr = offset; + element->length = buf->scatterlist.sgl->dma_length; + element->segl_addr = seg->dma_start; + element->segl_length = seg->page_count * PAGE_SIZE; + return 0; +} \ No newline at end of file diff --git a/drivers/staging/apple-bce/queue_dma.h b/drivers/staging/apple-bce/queue_dma.h new file mode 100644 index 000000000000..f8a57e50e7a3 --- /dev/null +++ b/drivers/staging/apple-bce/queue_dma.h @@ -0,0 +1,50 @@ +#ifndef BCE_QUEUE_DMA_H +#define BCE_QUEUE_DMA_H + +#include + +struct bce_qe_submission; + +struct bce_segment_list_header { + u64 element_count; + u64 data_size; + + u64 next_segl_addr; + u64 next_segl_length; +}; +struct bce_segment_list_element { + u64 addr; + u64 length; +}; + +struct bce_segment_list_element_hostinfo { + struct bce_segment_list_element_hostinfo *next; + void *page_start; + size_t page_count; + dma_addr_t dma_start; +}; + + +struct bce_dma_buffer { + enum dma_data_direction direction; + struct sg_table scatterlist; + struct bce_segment_list_element_hostinfo *seglist_hostinfo; +}; + +/* NOTE: Takes ownership of the sg_table if it succeeds. Ownership is not transferred on failure. */ +int bce_map_dma_buffer(struct device *dev, struct bce_dma_buffer *buf, struct sg_table scatterlist, + enum dma_data_direction dir); + +/* Creates a buffer from virtual memory (vmalloc) */ +int bce_map_dma_buffer_vm(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir); + +/* Creates a buffer from kernel memory (kmalloc) */ +int bce_map_dma_buffer_km(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir); + +void bce_unmap_dma_buffer(struct device *dev, struct bce_dma_buffer *buf); + +int bce_set_submission_buf(struct bce_qe_submission *element, struct bce_dma_buffer *buf, size_t offset, size_t length); + +#endif //BCE_QUEUE_DMA_H diff --git a/drivers/staging/apple-bce/vhci/command.h b/drivers/staging/apple-bce/vhci/command.h new file mode 100644 index 000000000000..26619e0bccfa --- /dev/null +++ b/drivers/staging/apple-bce/vhci/command.h @@ -0,0 +1,204 @@ +#ifndef BCE_VHCI_COMMAND_H +#define BCE_VHCI_COMMAND_H + +#include "queue.h" +#include +#include + +#define BCE_VHCI_CMD_TIMEOUT_SHORT msecs_to_jiffies(2000) +#define BCE_VHCI_CMD_TIMEOUT_LONG msecs_to_jiffies(30000) + +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2 2 +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS (1 << BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2) + +typedef u8 bce_vhci_port_t; +typedef u8 bce_vhci_device_t; + +enum bce_vhci_command { + BCE_VHCI_CMD_CONTROLLER_ENABLE = 1, + BCE_VHCI_CMD_CONTROLLER_DISABLE = 2, + BCE_VHCI_CMD_CONTROLLER_START = 3, + BCE_VHCI_CMD_CONTROLLER_PAUSE = 4, + + BCE_VHCI_CMD_PORT_POWER_ON = 0x10, + BCE_VHCI_CMD_PORT_POWER_OFF = 0x11, + BCE_VHCI_CMD_PORT_RESUME = 0x12, + BCE_VHCI_CMD_PORT_SUSPEND = 0x13, + BCE_VHCI_CMD_PORT_RESET = 0x14, + BCE_VHCI_CMD_PORT_DISABLE = 0x15, + BCE_VHCI_CMD_PORT_STATUS = 0x16, + + BCE_VHCI_CMD_DEVICE_CREATE = 0x30, + BCE_VHCI_CMD_DEVICE_DESTROY = 0x31, + + BCE_VHCI_CMD_ENDPOINT_CREATE = 0x40, + BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x41, + BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x42, + BCE_VHCI_CMD_ENDPOINT_RESET = 0x44, + + /* Device to host only */ + BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE = 0x43, + BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000, + BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS = 0x1005 +}; + +enum bce_vhci_endpoint_state { + BCE_VHCI_ENDPOINT_ACTIVE = 0, + BCE_VHCI_ENDPOINT_PAUSED = 1, + BCE_VHCI_ENDPOINT_STALLED = 2 +}; + +static inline int bce_vhci_cmd_controller_enable(struct bce_vhci_command_queue *q, u8 busNum, u16 *portMask) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE; + cmd.param1 = 0x7100u | busNum; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); + if (!status) + *portMask = (u16) res.param2; + return status; +} +static inline int bce_vhci_cmd_controller_disable(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_start(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_pause(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_PAUSE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_port_power_on(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_power_off(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_OFF; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_resume(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESUME; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_suspend(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_SUSPEND; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_reset(struct bce_vhci_command_queue *q, bce_vhci_port_t port, u32 timeout) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESET; + cmd.param1 = port; + cmd.param2 = timeout; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_disable(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_DISABLE; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_status(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + u32 clearFlags, u32 *resStatus) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; + cmd.param1 = port; + cmd.param2 = clearFlags & 0x560000; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status >= 0) + *resStatus = (u32) res.param2; + return status; +} + +static inline int bce_vhci_cmd_device_create(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + bce_vhci_device_t *dev) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE; + cmd.param1 = port; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (!status) + *dev = (bce_vhci_device_t) res.param2; + return status; +} +static inline int bce_vhci_cmd_device_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = dev; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_endpoint_create(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, + struct usb_endpoint_descriptor *desc) +{ + struct bce_vhci_message cmd, res; + int endpoint_type = usb_endpoint_type(desc); + int maxp = usb_endpoint_maxp(desc); + int maxp_burst = usb_endpoint_maxp_mult(desc) * maxp; + u8 max_active_requests_pow2 = 0; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE; + cmd.param1 = dev | ((desc->bEndpointAddress & 0x8Fu) << 8); + if (endpoint_type == USB_ENDPOINT_XFER_BULK) + max_active_requests_pow2 = BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2; + cmd.param2 = endpoint_type | ((max_active_requests_pow2 & 0xf) << 4) | (maxp << 16) | ((u64) maxp_burst << 32); + if (endpoint_type == USB_ENDPOINT_XFER_INT) + cmd.param2 |= (desc->bInterval - 1) << 8; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_set_state(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint, + enum bce_vhci_endpoint_state newState, enum bce_vhci_endpoint_state *retState) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_SET_STATE; + cmd.param1 = dev | (endpoint << 8); + cmd.param2 = (u64) newState; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status != BCE_VHCI_INTERNAL_ERROR && status != BCE_VHCI_NO_POWER) + *retState = (enum bce_vhci_endpoint_state) res.param2; + return status; +} +static inline int bce_vhci_cmd_endpoint_reset(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} + + +#endif //BCE_VHCI_COMMAND_H diff --git a/drivers/staging/apple-bce/vhci/queue.c b/drivers/staging/apple-bce/vhci/queue.c new file mode 100644 index 000000000000..7b0b5027157b --- /dev/null +++ b/drivers/staging/apple-bce/vhci/queue.c @@ -0,0 +1,268 @@ +#include "queue.h" +#include "vhci.h" +#include "../apple_bce.h" + + +static void bce_vhci_message_queue_completion(struct bce_queue_sq *sq); + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name) +{ + int status; + ret->cq = bce_create_cq(vhci->dev, VHCI_EVENT_QUEUE_EL_COUNT); + if (!ret->cq) + return -EINVAL; + ret->sq = bce_create_sq(vhci->dev, ret->cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_TO_DEVICE, + bce_vhci_message_queue_completion, ret); + if (!ret->sq) { + status = -EINVAL; + goto fail_cq; + } + ret->data = dma_alloc_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + status = -EINVAL; + goto fail_sq; + } + return 0; + +fail_sq: + bce_destroy_sq(vhci->dev, ret->sq); + ret->sq = NULL; +fail_cq: + bce_destroy_cq(vhci->dev, ret->cq); + ret->cq = NULL; + return status; +} + +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q) +{ + if (!q->cq) + return; + dma_free_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + bce_destroy_sq(vhci->dev, q->sq); + bce_destroy_cq(vhci->dev, q->cq); +} + +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req) +{ + int sidx; + struct bce_qe_submission *s; + sidx = q->sq->tail; + s = bce_next_submission(q->sq); + pr_debug("bce-vhci: Send message: %x s=%x p1=%x p2=%llx\n", req->cmd, req->status, req->param1, req->param2); + q->data[sidx] = *req; + bce_set_submission_single(s, q->dma_addr + sizeof(struct bce_vhci_message) * sidx, + sizeof(struct bce_vhci_message)); + bce_submit_to_device(q->sq); +} + +static void bce_vhci_message_queue_completion(struct bce_queue_sq *sq) +{ + while (bce_next_completion(sq)) + bce_notify_submission_complete(sq); +} + + + +static void bce_vhci_event_queue_completion(struct bce_queue_sq *sq); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_sq_completion compl) +{ + ret->vhci = vhci; + + ret->sq = bce_create_sq(vhci->dev, vhci->ev_cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_FROM_DEVICE, compl, ret); + if (!ret->sq) + return -EINVAL; + ret->data = dma_alloc_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + bce_destroy_sq(vhci->dev, ret->sq); + ret->sq = NULL; + return -EINVAL; + } + + init_completion(&ret->queue_empty_completion); + bce_vhci_event_queue_submit_pending(ret, VHCI_EVENT_PENDING_COUNT); + return 0; +} + +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb) +{ + ret->cb = cb; + return __bce_vhci_event_queue_create(vhci, ret, name, bce_vhci_event_queue_completion); +} + +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q) +{ + if (!q->sq) + return; + dma_free_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + bce_destroy_sq(vhci->dev, q->sq); +} + +static void bce_vhci_event_queue_completion(struct bce_queue_sq *sq) +{ + struct bce_sq_completion_data *cd; + struct bce_vhci_event_queue *ev = sq->userdata; + struct bce_vhci_message *msg; + size_t cnt = 0; + + while ((cd = bce_next_completion(sq))) { + if (cd->status == BCE_COMPLETION_ABORTED) { /* We flushed the queue */ + bce_notify_submission_complete(sq); + continue; + } + msg = &ev->data[sq->head]; + pr_debug("bce-vhci: Got event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + ev->cb(ev, msg); + + bce_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(ev, cnt); + if (atomic_read(&sq->available_commands) == sq->el_count - 1) + complete(&ev->queue_empty_completion); +} + +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count) +{ + int idx; + struct bce_qe_submission *s; + while (count--) { + if (bce_reserve_submission(q->sq, NULL)) { + pr_err("bce-vhci: Failed to reserve an event queue submission\n"); + break; + } + idx = q->sq->tail; + s = bce_next_submission(q->sq); + bce_set_submission_single(s, + q->dma_addr + idx * sizeof(struct bce_vhci_message), sizeof(struct bce_vhci_message)); + } + bce_submit_to_device(q->sq); +} + +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q) +{ + unsigned long timeout; + reinit_completion(&q->queue_empty_completion); + if (bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, q->sq->qid)) + pr_warn("bce-vhci: failed to flush event queue\n"); + timeout = msecs_to_jiffies(5000); + while (atomic_read(&q->sq->available_commands) != q->sq->el_count - 1) { + timeout = wait_for_completion_timeout(&q->queue_empty_completion, timeout); + if (timeout == 0) { + pr_err("bce-vhci: waiting for queue to be flushed timed out\n"); + break; + } + } +} + +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q) +{ + if (atomic_read(&q->sq->available_commands) != q->sq->el_count - 1) { + pr_err("bce-vhci: resume of a queue with pending submissions\n"); + return; + } + bce_vhci_event_queue_submit_pending(q, VHCI_EVENT_PENDING_COUNT); +} + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq) +{ + ret->mq = mq; + ret->completion.result = NULL; + init_completion(&ret->completion.completion); + spin_lock_init(&ret->completion_lock); + mutex_init(&ret->mutex); +} + +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq) +{ + spin_lock(&cq->completion_lock); + if (cq->completion.result) { + memset(cq->completion.result, 0, sizeof(struct bce_vhci_message)); + cq->completion.result->status = BCE_VHCI_ABORT; + complete(&cq->completion.completion); + cq->completion.result = NULL; + } + spin_unlock(&cq->completion_lock); + mutex_lock(&cq->mutex); + mutex_unlock(&cq->mutex); + mutex_destroy(&cq->mutex); +} + +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg) +{ + struct bce_vhci_command_queue_completion *c = &cq->completion; + + spin_lock(&cq->completion_lock); + if (c->result) { + *c->result = *msg; + complete(&c->completion); + c->result = NULL; + } + spin_unlock(&cq->completion_lock); +} + +static int __bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + struct bce_vhci_command_queue_completion *c; + struct bce_vhci_message creq; + c = &cq->completion; + + if ((status = bce_reserve_submission(cq->mq->sq, &timeout))) + return status; + + spin_lock(&cq->completion_lock); + c->result = res; + reinit_completion(&c->completion); + spin_unlock(&cq->completion_lock); + + bce_vhci_message_queue_write(cq->mq, req); + + if (!wait_for_completion_timeout(&c->completion, timeout)) { + /* we ran out of time, send cancellation */ + pr_debug("bce-vhci: command timed out req=%x\n", req->cmd); + if ((status = bce_reserve_submission(cq->mq->sq, &timeout))) + return status; + + creq = *req; + creq.cmd |= 0x4000; + bce_vhci_message_queue_write(cq->mq, &creq); + + if (!wait_for_completion_timeout(&c->completion, 1000)) { + pr_err("bce-vhci: Possible desync, cmd cancel timed out\n"); + + spin_lock(&cq->completion_lock); + c->result = NULL; + spin_unlock(&cq->completion_lock); + return -ETIMEDOUT; + } + if ((res->cmd & ~0x8000) == creq.cmd) + return -ETIMEDOUT; + /* reply for the previous command most likely arrived */ + } + + if ((res->cmd & ~0x8000) != req->cmd) { + pr_err("bce-vhci: Possible desync, cmd reply mismatch req=%x, res=%x\n", req->cmd, res->cmd); + return -EIO; + } + if (res->status == BCE_VHCI_SUCCESS) + return 0; + return res->status; +} + +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + mutex_lock(&cq->mutex); + status = __bce_vhci_command_queue_execute(cq, req, res, timeout); + mutex_unlock(&cq->mutex); + return status; +} diff --git a/drivers/staging/apple-bce/vhci/queue.h b/drivers/staging/apple-bce/vhci/queue.h new file mode 100644 index 000000000000..adb705b6ba1d --- /dev/null +++ b/drivers/staging/apple-bce/vhci/queue.h @@ -0,0 +1,76 @@ +#ifndef BCE_VHCI_QUEUE_H +#define BCE_VHCI_QUEUE_H + +#include +#include "../queue.h" + +#define VHCI_EVENT_QUEUE_EL_COUNT 256 +#define VHCI_EVENT_PENDING_COUNT 32 + +struct bce_vhci; +struct bce_vhci_event_queue; + +enum bce_vhci_message_status { + BCE_VHCI_SUCCESS = 1, + BCE_VHCI_ERROR = 2, + BCE_VHCI_USB_PIPE_STALL = 3, + BCE_VHCI_ABORT = 4, + BCE_VHCI_BAD_ARGUMENT = 5, + BCE_VHCI_OVERRUN = 6, + BCE_VHCI_INTERNAL_ERROR = 7, + BCE_VHCI_NO_POWER = 8, + BCE_VHCI_UNSUPPORTED = 9 +}; +struct bce_vhci_message { + u16 cmd; + u16 status; // bce_vhci_message_status + u32 param1; + u64 param2; +}; + +struct bce_vhci_message_queue { + struct bce_queue_cq *cq; + struct bce_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; +}; +typedef void (*bce_vhci_event_queue_callback)(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +struct bce_vhci_event_queue { + struct bce_vhci *vhci; + struct bce_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; + bce_vhci_event_queue_callback cb; + struct completion queue_empty_completion; +}; +struct bce_vhci_command_queue_completion { + struct bce_vhci_message *result; + struct completion completion; +}; +struct bce_vhci_command_queue { + struct bce_vhci_message_queue *mq; + struct bce_vhci_command_queue_completion completion; + struct spinlock completion_lock; + struct mutex mutex; +}; + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name); +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q); +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_sq_completion compl); +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb); +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count); +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q); + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq); +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq); +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout); +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg); + +#endif //BCE_VHCI_QUEUE_H diff --git a/drivers/staging/apple-bce/vhci/transfer.c b/drivers/staging/apple-bce/vhci/transfer.c new file mode 100644 index 000000000000..d77220707492 --- /dev/null +++ b/drivers/staging/apple-bce/vhci/transfer.c @@ -0,0 +1,673 @@ +#include "transfer.h" +#include "../queue.h" +#include "vhci.h" +#include "../apple_bce.h" +#include + +static void bce_vhci_transfer_queue_completion(struct bce_queue_sq *sq); +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q); +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q); + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb); +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg); +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c); + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work); + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir) +{ + char name[0x21]; + INIT_LIST_HEAD(&q->evq); + INIT_LIST_HEAD(&q->giveback_urb_list); + spin_lock_init(&q->urb_lock); + mutex_init(&q->pause_lock); + q->vhci = vhci; + q->endp = endp; + q->dev_addr = dev_addr; + q->endp_addr = (u8) (endp->desc.bEndpointAddress & 0x8F); + q->state = BCE_VHCI_ENDPOINT_ACTIVE; + q->active = true; + q->stalled = false; + q->max_active_requests = 1; + if (usb_endpoint_type(&endp->desc) == USB_ENDPOINT_XFER_BULK) + q->max_active_requests = BCE_VHCI_BULK_MAX_ACTIVE_URBS; + q->remaining_active_requests = q->max_active_requests; + q->cq = bce_create_cq(vhci->dev, 0x100); + INIT_WORK(&q->w_reset, bce_vhci_transfer_queue_reset_w); + q->sq_in = NULL; + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, 0x80 | usb_endpoint_num(&endp->desc)); + q->sq_in = bce_create_sq(vhci->dev, q->cq, name, 0x100, DMA_FROM_DEVICE, + bce_vhci_transfer_queue_completion, q); + } + q->sq_out = NULL; + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, usb_endpoint_num(&endp->desc)); + q->sq_out = bce_create_sq(vhci->dev, q->cq, name, 0x100, DMA_TO_DEVICE, + bce_vhci_transfer_queue_completion, q); + } +} + +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q) +{ + bce_vhci_transfer_queue_giveback(q); + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + bce_destroy_sq(vhci->dev, q->sq_in); + if (q->sq_out) + bce_destroy_sq(vhci->dev, q->sq_out); + bce_destroy_cq(vhci->dev, q->cq); +} + +static inline bool bce_vhci_transfer_queue_can_init_urb(struct bce_vhci_transfer_queue *q) +{ + return q->remaining_active_requests > 0; +} + +static void bce_vhci_transfer_queue_defer_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + struct bce_vhci_list_message *lm; + lm = kmalloc(sizeof(struct bce_vhci_list_message), GFP_KERNEL); + INIT_LIST_HEAD(&lm->list); + lm->msg = *msg; + list_add_tail(&lm->list, &q->evq); +} + +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + while (!list_empty(&q->giveback_urb_list)) { + urb = list_first_entry(&q->giveback_urb_list, struct urb, urb_list); + list_del(&urb->urb_list); + + spin_unlock_irqrestore(&q->urb_lock, flags); + usb_hcd_giveback_urb(q->vhci->hcd, urb, urb->status); + spin_lock_irqsave(&q->urb_lock, flags); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q); + +static void bce_vhci_transfer_queue_deliver_pending(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb; + struct bce_vhci_list_message *lm; + + while (!list_empty(&q->endp->urb_list) && !list_empty(&q->evq)) { + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + if (bce_vhci_urb_update(urb->hcpriv, &lm->msg) == -EAGAIN) + break; + list_del(&lm->list); + kfree(lm); + } + + /* some of the URBs could have been completed, so initialize more URBs if possible */ + bce_vhci_transfer_queue_init_pending_urbs(q); +} + +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct bce_vhci_list_message *lm; + spin_lock_irqsave(&q->urb_lock, flags); + while (!list_empty(&q->evq)) { + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + list_del(&lm->list); + kfree(lm); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + unsigned long flags; + struct bce_vhci_urb *turb; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + bce_vhci_transfer_queue_deliver_pending(q); + + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && + (!list_empty(&q->evq) || list_empty(&q->endp->urb_list))) { + bce_vhci_transfer_queue_defer_event(q, msg); + goto complete; + } + if (list_empty(&q->endp->urb_list)) { + pr_err("bce-vhci: [%02x] Unexpected transfer queue event\n", q->endp_addr); + goto complete; + } + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + turb = urb->hcpriv; + if (bce_vhci_urb_update(turb, msg) == -EAGAIN) { + bce_vhci_transfer_queue_defer_event(q, msg); + } else { + bce_vhci_transfer_queue_init_pending_urbs(q); + } + +complete: + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +static void bce_vhci_transfer_queue_completion(struct bce_queue_sq *sq) +{ + unsigned long flags; + struct bce_sq_completion_data *c; + struct urb *urb; + struct bce_vhci_transfer_queue *q = sq->userdata; + spin_lock_irqsave(&q->urb_lock, flags); + while ((c = bce_next_completion(sq))) { + if (c->status == BCE_COMPLETION_ABORTED) { /* We flushed the queue */ + pr_debug("bce-vhci: [%02x] Got an abort completion\n", q->endp_addr); + bce_notify_submission_complete(sq); + continue; + } + if (list_empty(&q->endp->urb_list)) { + pr_err("bce-vhci: [%02x] Got a completion while no requests are pending\n", q->endp_addr); + continue; + } + pr_debug("bce-vhci: [%02x] Got a transfer queue completion\n", q->endp_addr); + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + bce_vhci_urb_transfer_completion(urb->hcpriv, c); + bce_notify_submission_complete(sq); + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + spin_lock_irqsave(&q->urb_lock, flags); + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + if (q->sq_out) { + pr_err("bce-vhci: Not implemented: wait for pending output requests\n"); + } + bce_vhci_transfer_queue_remove_pending(q); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) + return status; + if (q->state != BCE_VHCI_ENDPOINT_PAUSED) + return -EINVAL; + if (q->sq_in) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_in->qid); + if (q->sq_out) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_out->qid); + return 0; +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb); + +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_ACTIVE, &q->state))) + return status; + if (q->state != BCE_VHCI_ENDPOINT_ACTIVE) + return -EINVAL; + spin_lock_irqsave(&q->urb_lock, flags); + q->active = true; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) { + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + bce_vhci_urb_init(vurb); + } else { + bce_vhci_urb_resume(vurb); + } + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + return 0; +} + +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + mutex_lock(&q->pause_lock); + if ((q->paused_by & src) != src) { + if (!q->paused_by) + ret = bce_vhci_transfer_queue_do_pause(q); + if (!ret) + q->paused_by |= src; + } + mutex_unlock(&q->pause_lock); + return ret; +} + +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + mutex_lock(&q->pause_lock); + if (q->paused_by & src) { + if (!(q->paused_by & ~src)) + ret = bce_vhci_transfer_queue_do_resume(q); + if (!ret) + q->paused_by &= ~src; + } + mutex_unlock(&q->pause_lock); + return ret; +} + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work) +{ + unsigned long flags; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_reset); + + mutex_lock(&q->pause_lock); + spin_lock_irqsave(&q->urb_lock, flags); + if (!q->stalled) { + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + return; + } + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + q->paused_by |= BCE_VHCI_PAUSE_INTERNAL_WQ; + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_in->qid); + if (q->sq_out) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_out->qid); + bce_vhci_cmd_endpoint_reset(&q->vhci->cq, q->dev_addr, (u8) (q->endp->desc.bEndpointAddress & 0x8F)); + spin_lock_irqsave(&q->urb_lock, flags); + q->stalled = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); +} + +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q) +{ + queue_work(q->vhci->tq_state_wq, &q->w_reset); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) + bce_vhci_urb_init(vurb); + } +} + + + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout); + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb) +{ + unsigned long flags; + int status = 0; + struct bce_vhci_urb *vurb; + vurb = kzalloc(sizeof(struct bce_vhci_urb), GFP_KERNEL); + urb->hcpriv = vurb; + + vurb->q = q; + vurb->urb = urb; + vurb->dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + vurb->is_control = (usb_endpoint_num(&urb->ep->desc) == 0); + + spin_lock_irqsave(&q->urb_lock, flags); + status = usb_hcd_link_urb_to_ep(q->vhci->hcd, urb); + if (status) { + spin_unlock_irqrestore(&q->urb_lock, flags); + urb->hcpriv = NULL; + kfree(vurb); + return status; + } + + if (q->active) { + if (bce_vhci_transfer_queue_can_init_urb(vurb->q)) + status = bce_vhci_urb_init(vurb); + else + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } else { + if (q->stalled) + bce_vhci_transfer_queue_request_reset(q); + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } + if (status) { + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + urb->hcpriv = NULL; + kfree(vurb); + } else { + bce_vhci_transfer_queue_deliver_pending(q); + } + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("bce-vhci: [%02x] URB enqueued (dir = %s, size = %i)\n", q->endp_addr, + usb_urb_dir_in(urb) ? "IN" : "OUT", urb->transfer_buffer_length); + return status; +} + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb) +{ + int status = 0; + + if (vurb->q->remaining_active_requests == 0) { + pr_err("bce-vhci: cannot init request (remaining_active_requests = 0)\n"); + return -EINVAL; + } + + if (vurb->is_control) { + vurb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST; + } else { + status = bce_vhci_urb_data_start(vurb, NULL); + } + + if (!status) { + --vurb->q->remaining_active_requests; + } + return status; +} + +static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->q; + struct bce_vhci *vhci = q->vhci; + struct urb *real_urb = urb->urb; + pr_debug("bce-vhci: [%02x] URB complete %i\n", q->endp_addr, status); + usb_hcd_unlink_urb_from_ep(vhci->hcd, real_urb); + real_urb->hcpriv = NULL; + real_urb->status = status; + if (urb->state != BCE_VHCI_URB_INIT_PENDING) + ++urb->q->remaining_active_requests; + kfree(urb); + list_add_tail(&real_urb->urb_list, &q->giveback_urb_list); +} + +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status) +{ + struct bce_vhci_urb *vurb; + unsigned long flags; + int ret; + enum bce_vhci_urb_state old_state; + + spin_lock_irqsave(&q->urb_lock, flags); + if ((ret = usb_hcd_check_unlink_urb(q->vhci->hcd, urb, status))) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return ret; + } + + vurb = urb->hcpriv; + + old_state = vurb->state; /* save old state to use later because we'll set state as cancelled */ + + if (old_state == BCE_VHCI_URB_CANCELLED) { + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("bce-vhci: URB %p is already cancelled, skipping\n", urb); + return 0; + } + + vurb->state = BCE_VHCI_URB_CANCELLED; + + /* If the URB wasn't posted to the device yet, we can still remove it on the host without pausing the queue. */ + if (old_state != BCE_VHCI_URB_INIT_PENDING) { + pr_debug("bce-vhci: [%02x] Cancelling URB\n", q->endp_addr); + + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_pause(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + spin_lock_irqsave(&q->urb_lock, flags); + + ++q->remaining_active_requests; + } + + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + + spin_unlock_irqrestore(&q->urb_lock, flags); + + usb_hcd_giveback_urb(q->vhci->hcd, urb, status); + + if (old_state != BCE_VHCI_URB_INIT_PENDING) + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + + kfree(vurb); + + return 0; +} + +static int bce_vhci_urb_data_transfer_in(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + struct bce_vhci_message msg; + struct bce_qe_submission *s; + u32 tr_len; + int reservation1, reservation2 = -EFAULT; + + pr_debug("bce-vhci: [%02x] DMA from device %llx %x\n", urb->q->endp_addr, + (u64) urb->urb->transfer_dma, urb->urb->transfer_buffer_length); + + /* Reserve both a message and a submission, so we don't run into issues later. */ + reservation1 = bce_reserve_submission(urb->q->vhci->msg_asynchronous.sq, timeout); + if (!reservation1) + reservation2 = bce_reserve_submission(urb->q->sq_in, timeout); + if (reservation1 || reservation2) { + pr_err("bce-vhci: Failed to reserve a submission for URB data transfer\n"); + if (!reservation1) + bce_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); + return -ENOMEM; + } + + urb->send_offset = urb->receive_offset; + + tr_len = urb->urb->transfer_buffer_length - urb->send_offset; + + spin_lock(&urb->q->vhci->msg_asynchronous_lock); + msg.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + msg.status = 0; + msg.param1 = ((urb->urb->ep->desc.bEndpointAddress & 0x8Fu) << 8) | urb->q->dev_addr; + msg.param2 = tr_len; + bce_vhci_message_queue_write(&urb->q->vhci->msg_asynchronous, &msg); + spin_unlock(&urb->q->vhci->msg_asynchronous_lock); + + s = bce_next_submission(urb->q->sq_in); + bce_set_submission_single(s, urb->urb->transfer_dma + urb->send_offset, tr_len); + bce_submit_to_device(urb->q->sq_in); + + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + return 0; +} + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + if (urb->dir == DMA_TO_DEVICE) { + if (urb->urb->transfer_buffer_length > 0) + urb->state = BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST; + else + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + return 0; + } else { + return bce_vhci_urb_data_transfer_in(urb, timeout); + } +} + +static int bce_vhci_urb_send_out_data(struct bce_vhci_urb *urb, dma_addr_t addr, size_t size) +{ + struct bce_qe_submission *s; + unsigned long timeout = 0; + if (bce_reserve_submission(urb->q->sq_out, &timeout)) { + pr_err("bce-vhci: Failed to reserve a submission for URB data transfer\n"); + return -EPIPE; + } + + pr_debug("bce-vhci: [%02x] DMA to device %llx %lx\n", urb->q->endp_addr, (u64) addr, size); + + s = bce_next_submission(urb->q->sq_out); + bce_set_submission_single(s, addr, size); + bce_submit_to_device(urb->q->sq_out); + return 0; +} + +static int bce_vhci_urb_data_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + u32 tr_len; + int status; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + tr_len = min(urb->urb->transfer_buffer_length - urb->send_offset, (u32) msg->param2); + if ((status = bce_vhci_urb_send_out_data(urb, urb->urb->transfer_dma + urb->send_offset, tr_len))) + return status; + urb->send_offset += tr_len; + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + return 0; + } + } + + /* 0x1000 in out queues aren't really unexpected */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("bce-vhci: [%02x] %s URB unexpected message (state = %x, msg: %x %x %x %llx)\n", + urb->q->endp_addr, (urb->is_control ? "Control (data update)" : "Data"), urb->state, + msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_data_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) +{ + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + urb->receive_offset += c->data_size; + if (urb->dir == DMA_FROM_DEVICE || urb->receive_offset >= urb->urb->transfer_buffer_length) { + urb->urb->actual_length = (u32) urb->receive_offset; + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + if (!urb->is_control) { + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + } + } else { + pr_err("bce-vhci: [%02x] Data URB unexpected completion\n", urb->q->endp_addr); + } + return 0; +} + + +static int bce_vhci_urb_control_check_status(struct bce_vhci_urb *urb) +{ + struct bce_vhci_transfer_queue *q = urb->q; + if (urb->received_status == 0) + return 0; + if (urb->state == BCE_VHCI_URB_DATA_TRANSFER_COMPLETE || + (urb->received_status != BCE_VHCI_SUCCESS && urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST && + urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION)) { + urb->state = BCE_VHCI_URB_CONTROL_COMPLETE; + if (urb->received_status != BCE_VHCI_SUCCESS) { + pr_err("bce-vhci: [%02x] URB failed: %x\n", urb->q->endp_addr, urb->received_status); + urb->q->active = false; + urb->q->stalled = true; + bce_vhci_urb_complete(urb, -EPIPE); + if (!list_empty(&q->endp->urb_list)) + bce_vhci_transfer_queue_request_reset(q); + return -ENOENT; + } + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + return 0; +} + +static int bce_vhci_urb_control_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + int status; + if (msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + urb->received_status = msg->status; + return bce_vhci_urb_control_check_status(urb); + } + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + if (bce_vhci_urb_send_out_data(urb, urb->urb->setup_dma, sizeof(struct usb_ctrlrequest))) { + pr_err("bce-vhci: [%02x] Failed to start URB setup transfer\n", urb->q->endp_addr); + return 0; /* TODO: fail the URB? */ + } + urb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION; + pr_debug("bce-vhci: [%02x] Sent setup %llx\n", urb->q->endp_addr, urb->urb->setup_dma); + return 0; + } + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_update(urb, msg))) + return status; + return bce_vhci_urb_control_check_status(urb); + } + + /* 0x1000 in out queues aren't really unexpected */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("bce-vhci: [%02x] Control URB unexpected message (state = %x, msg: %x %x %x %llx)\n", urb->q->endp_addr, + urb->state, msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_control_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) +{ + int status; + unsigned long timeout; + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION) { + if (c->data_size != sizeof(struct usb_ctrlrequest)) + pr_err("bce-vhci: [%02x] transfer complete data size mistmatch for usb_ctrlrequest (%llx instead of %lx)\n", + urb->q->endp_addr, c->data_size, sizeof(struct usb_ctrlrequest)); + + timeout = 1000; + status = bce_vhci_urb_data_start(urb, &timeout); + if (status) { + bce_vhci_urb_complete(urb, status); + return -ENOENT; + } + return 0; + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_transfer_completion(urb, c))) + return status; + return bce_vhci_urb_control_check_status(urb); + } else { + pr_err("bce-vhci: [%02x] Control URB unexpected completion (state = %x)\n", urb->q->endp_addr, urb->state); + } + return 0; +} + +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + if (urb->state == BCE_VHCI_URB_INIT_PENDING) + return -EAGAIN; + if (urb->is_control) + return bce_vhci_urb_control_update(urb, msg); + else + return bce_vhci_urb_data_update(urb, msg); +} + +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) +{ + if (urb->is_control) + return bce_vhci_urb_control_transfer_completion(urb, c); + else + return bce_vhci_urb_data_transfer_completion(urb, c); +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb) +{ + int status = 0; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + status = bce_vhci_urb_data_transfer_in(urb, NULL); + } + if (status) + bce_vhci_urb_complete(urb, status); +} diff --git a/drivers/staging/apple-bce/vhci/transfer.h b/drivers/staging/apple-bce/vhci/transfer.h new file mode 100644 index 000000000000..b5403d5703ed --- /dev/null +++ b/drivers/staging/apple-bce/vhci/transfer.h @@ -0,0 +1,75 @@ +#ifndef BCEDRIVER_TRANSFER_H +#define BCEDRIVER_TRANSFER_H + +#include +#include "queue.h" +#include "command.h" +#include "../queue.h" + +struct bce_vhci_list_message { + struct list_head list; + struct bce_vhci_message msg; +}; +enum bce_vhci_pause_source { + BCE_VHCI_PAUSE_INTERNAL_WQ = 1, + BCE_VHCI_PAUSE_FIRMWARE = 2, + BCE_VHCI_PAUSE_SUSPEND = 4, + BCE_VHCI_PAUSE_SHUTDOWN = 8 +}; +struct bce_vhci_transfer_queue { + struct bce_vhci *vhci; + struct usb_host_endpoint *endp; + enum bce_vhci_endpoint_state state; + u32 max_active_requests, remaining_active_requests; + bool active, stalled; + u32 paused_by; + bce_vhci_device_t dev_addr; + u8 endp_addr; + struct bce_queue_cq *cq; + struct bce_queue_sq *sq_in; + struct bce_queue_sq *sq_out; + struct list_head evq; + struct spinlock urb_lock; + struct mutex pause_lock; + struct list_head giveback_urb_list; + + struct work_struct w_reset; +}; +enum bce_vhci_urb_state { + BCE_VHCI_URB_INIT_PENDING, + + BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST, + BCE_VHCI_URB_WAITING_FOR_COMPLETION, + BCE_VHCI_URB_DATA_TRANSFER_COMPLETE, + + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST, + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION, + BCE_VHCI_URB_CONTROL_COMPLETE, + + BCE_VHCI_URB_CANCELLED +}; +struct bce_vhci_urb { + struct urb *urb; + struct bce_vhci_transfer_queue *q; + enum dma_data_direction dir; + bool is_control; + enum bce_vhci_urb_state state; + int received_status; + u32 send_offset; + u32 receive_offset; +}; + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir); +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q); +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg); +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q); + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb); +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status); + +#endif //BCEDRIVER_TRANSFER_H diff --git a/drivers/staging/apple-bce/vhci/vhci.c b/drivers/staging/apple-bce/vhci/vhci.c new file mode 100644 index 000000000000..3394e0aa2eef --- /dev/null +++ b/drivers/staging/apple-bce/vhci/vhci.c @@ -0,0 +1,763 @@ +#include "vhci.h" +#include "../apple_bce.h" +#include "command.h" +#include +#include +#include +#include + +static dev_t bce_vhci_chrdev; +static struct class *bce_vhci_class; +static const struct hc_driver bce_vhci_driver; +static u16 bce_vhci_port_mask = U16_MAX; + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci); +static int bce_vhci_create_message_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci); +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws); +static void bce_vhci_firmware_event_completion(struct bce_queue_sq *sq); + +int bce_vhci_create(struct apple_bce_device *dev, struct bce_vhci *vhci) +{ + int status; + + spin_lock_init(&vhci->hcd_spinlock); + + vhci->dev = dev; + + vhci->vdevt = bce_vhci_chrdev; + vhci->vdev = device_create(bce_vhci_class, dev->dev, vhci->vdevt, NULL, "bce-vhci"); + if (IS_ERR_OR_NULL(vhci->vdev)) { + status = PTR_ERR(vhci->vdev); + goto fail_dev; + } + + if ((status = bce_vhci_create_message_queues(vhci))) + goto fail_mq; + if ((status = bce_vhci_create_event_queues(vhci))) + goto fail_eq; + + vhci->tq_state_wq = alloc_ordered_workqueue("bce-vhci-tq-state", 0); + INIT_WORK(&vhci->w_fw_events, bce_vhci_handle_firmware_events_w); + + vhci->hcd = usb_create_hcd(&bce_vhci_driver, vhci->vdev, "bce-vhci"); + if (!vhci->hcd) { + status = -ENOMEM; + goto fail_hcd; + } + vhci->hcd->self.sysdev = &dev->pci->dev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + vhci->hcd->self.uses_dma = 1; +#endif + *((struct bce_vhci **) vhci->hcd->hcd_priv) = vhci; + vhci->hcd->speed = HCD_USB2; + + if ((status = usb_add_hcd(vhci->hcd, 0, 0))) + goto fail_hcd; + + return 0; + +fail_hcd: + bce_vhci_destroy_event_queues(vhci); +fail_eq: + bce_vhci_destroy_message_queues(vhci); +fail_mq: + device_destroy(bce_vhci_class, vhci->vdevt); +fail_dev: + if (!status) + status = -EINVAL; + return status; +} + +void bce_vhci_destroy(struct bce_vhci *vhci) +{ + usb_remove_hcd(vhci->hcd); + bce_vhci_destroy_event_queues(vhci); + bce_vhci_destroy_message_queues(vhci); + device_destroy(bce_vhci_class, vhci->vdevt); +} + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd) +{ + return *((struct bce_vhci **) hcd->hcd_priv); +} + +int bce_vhci_start(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + u16 port_mask = 0; + bce_vhci_port_t port_no = 0; + if ((status = bce_vhci_cmd_controller_enable(&vhci->cq, 1, &port_mask))) + return status; + vhci->port_mask = port_mask; + vhci->port_power_mask = 0; + if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) + return status; + port_mask = vhci->port_mask; + while (port_mask) { + port_no += 1; + port_mask >>= 1; + } + vhci->port_count = port_no; + return 0; +} + +void bce_vhci_stop(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_cmd_controller_disable(&vhci->cq); +} + +static int bce_vhci_hub_status_data(struct usb_hcd *hcd, char *buf) +{ + return 0; +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout); + +static int bce_vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + struct usb_hub_descriptor *hd; + struct usb_hub_status *hs; + struct usb_port_status *ps; + u32 port_status; + // pr_info("bce-vhci: bce_vhci_hub_control %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); + if (typeReq == GetHubDescriptor && wLength >= sizeof(struct usb_hub_descriptor)) { + hd = (struct usb_hub_descriptor *) buf; + memset(hd, 0, sizeof(*hd)); + hd->bDescLength = sizeof(struct usb_hub_descriptor); + hd->bDescriptorType = USB_DT_HUB; + hd->bNbrPorts = (u8) vhci->port_count; + hd->wHubCharacteristics = HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_INDV_PORT_OCPM; + hd->bPwrOn2PwrGood = 0; + hd->bHubContrCurrent = 0; + return 0; + } else if (typeReq == GetHubStatus && wLength >= sizeof(struct usb_hub_status)) { + hs = (struct usb_hub_status *) buf; + memset(hs, 0, sizeof(*hs)); + hs->wHubStatus = 0; + hs->wHubChange = 0; + return 0; + } else if (typeReq == GetPortStatus && wLength >= 4 /* usb 2.0 */) { + ps = (struct usb_port_status *) buf; + ps->wPortStatus = 0; + ps->wPortChange = 0; + + if (vhci->port_power_mask & BIT(wIndex)) + ps->wPortStatus |= USB_PORT_STAT_POWER; + + if (!(bce_vhci_port_mask & BIT(wIndex))) + return 0; + + if ((status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0, &port_status))) + return status; + + if (port_status & 16) + ps->wPortStatus |= USB_PORT_STAT_ENABLE | USB_PORT_STAT_HIGH_SPEED; + if (port_status & 4) + ps->wPortStatus |= USB_PORT_STAT_CONNECTION; + if (port_status & 2) + ps->wPortStatus |= USB_PORT_STAT_OVERCURRENT; + if (port_status & 8) + ps->wPortStatus |= USB_PORT_STAT_RESET; + if (port_status & 0x60) + ps->wPortStatus |= USB_PORT_STAT_SUSPEND; + + if (port_status & 0x40000) + ps->wPortChange |= USB_PORT_STAT_C_CONNECTION; + + pr_debug("bce-vhci: Translated status %x to %x:%x\n", port_status, ps->wPortStatus, ps->wPortChange); + return 0; + } else if (typeReq == SetPortFeature) { + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) wIndex); + /* As far as I am aware, power status is not part of the port status so store it separately */ + if (!status) + vhci->port_power_mask |= BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_RESET) { + return bce_vhci_reset_device(vhci, wIndex, wValue); + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + /* TODO: Am I supposed to also suspend the endpoints? */ + pr_debug("bce-vhci: Suspending port %i\n", wIndex); + return bce_vhci_cmd_port_suspend(&vhci->cq, (u8) wIndex); + } + } else if (typeReq == ClearPortFeature) { + if (wValue == USB_PORT_FEAT_ENABLE) + return bce_vhci_cmd_port_disable(&vhci->cq, (u8) wIndex); + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) wIndex); + if (!status) + vhci->port_power_mask &= ~BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_C_CONNECTION) + return bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &port_status); + if (wValue == USB_PORT_FEAT_C_RESET) { /* I don't think I can transfer it in any way */ + return 0; + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + pr_debug("bce-vhci: Resuming port %i\n", wIndex); + return bce_vhci_cmd_port_resume(&vhci->cq, (u8) wIndex); + } + } + pr_err("bce-vhci: bce_vhci_hub_control unhandled request: %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); + dump_stack(); + return -EIO; +} + +static int bce_vhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + struct bce_vhci_device *vdev; + bce_vhci_device_t devid; + pr_info("bce_vhci_enable_device\n"); + + if (vhci->port_to_device[udev->portnum]) + return 0; + + /* We need to early address the device */ + if (bce_vhci_cmd_device_create(&vhci->cq, udev->portnum, &devid)) + return -EIO; + + pr_info("bce_vhci_cmd_device_create %i -> %i\n", udev->portnum, devid); + + vdev = kzalloc(sizeof(struct bce_vhci_device), GFP_KERNEL); + vhci->port_to_device[udev->portnum] = devid; + vhci->devices[devid] = vdev; + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[0], &udev->ep0, devid, DMA_BIDIRECTIONAL); + udev->ep0.hcpriv = &vdev->tq[0]; + vdev->tq_mask |= BIT(0); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &udev->ep0.desc); + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,8,0) +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev) +#else +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, unsigned int timeout_ms) //TODO: follow timeout +#endif +{ + /* This is the same as enable_device, but instead in the old scheme */ + return bce_vhci_enable_device(hcd, udev); +} + +static void bce_vhci_free_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int i; + bce_vhci_device_t devid; + struct bce_vhci_device *dev; + pr_info("bce_vhci_free_device %i\n", udev->portnum); + if (!vhci->port_to_device[udev->portnum]) + return; + devid = vhci->port_to_device[udev->portnum]; + dev = vhci->devices[devid]; + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) i); + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[udev->portnum] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + kfree(dev); +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout) +{ + struct bce_vhci_device *dev = NULL; + bce_vhci_device_t devid; + int i; + int status; + enum dma_data_direction dir; + pr_info("bce_vhci_reset_device %i\n", index); + + devid = vhci->port_to_device[index]; + if (devid) { + dev = vhci->devices[devid]; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) i); + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[index] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + } + status = bce_vhci_cmd_port_reset(&vhci->cq, (u8) index, timeout); + + if (dev) { + if ((status = bce_vhci_cmd_device_create(&vhci->cq, index, &devid))) + return status; + vhci->devices[devid] = dev; + vhci->port_to_device[index] = devid; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + dir = usb_endpoint_dir_in(&dev->tq[i].endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + if (i == 0) + dir = DMA_BIDIRECTIONAL; + bce_vhci_create_transfer_queue(vhci, &dev->tq[i], dev->tq[i].endp, devid, dir); + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &dev->tq[i].endp->desc); + } + } + } + + return status; +} + +static int bce_vhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) +{ + return 0; +} + +static int bce_vhci_get_frame_number(struct usb_hcd *hcd) +{ + return 0; +} + +static int bce_vhci_bus_suspend(struct usb_hcd *hcd) +{ + int i, j; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + pr_info("bce_vhci: suspend started\n"); + + pr_info("bce_vhci: suspend endpoints\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + for (j = 0; j < 32; j++) { + if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) + continue; + bce_vhci_transfer_queue_pause(&vhci->devices[vhci->port_to_device[i]]->tq[j], + BCE_VHCI_PAUSE_SUSPEND); + } + } + + pr_info("bce_vhci: suspend ports\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + bce_vhci_cmd_port_suspend(&vhci->cq, i); + } + pr_info("bce_vhci: suspend controller\n"); + if ((status = bce_vhci_cmd_controller_pause(&vhci->cq))) + return status; + + bce_vhci_event_queue_pause(&vhci->ev_commands); + bce_vhci_event_queue_pause(&vhci->ev_system); + bce_vhci_event_queue_pause(&vhci->ev_isochronous); + bce_vhci_event_queue_pause(&vhci->ev_interrupt); + bce_vhci_event_queue_pause(&vhci->ev_asynchronous); + pr_info("bce_vhci: suspend done\n"); + return 0; +} + +static int bce_vhci_bus_resume(struct usb_hcd *hcd) +{ + int i, j; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + pr_info("bce_vhci: resume started\n"); + + bce_vhci_event_queue_resume(&vhci->ev_system); + bce_vhci_event_queue_resume(&vhci->ev_isochronous); + bce_vhci_event_queue_resume(&vhci->ev_interrupt); + bce_vhci_event_queue_resume(&vhci->ev_asynchronous); + bce_vhci_event_queue_resume(&vhci->ev_commands); + + pr_info("bce_vhci: resume controller\n"); + if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) + return status; + + pr_info("bce_vhci: resume ports\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + bce_vhci_cmd_port_resume(&vhci->cq, i); + } + pr_info("bce_vhci: resume endpoints\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + for (j = 0; j < 32; j++) { + if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) + continue; + bce_vhci_transfer_queue_resume(&vhci->devices[vhci->port_to_device[i]]->tq[j], + BCE_VHCI_PAUSE_SUSPEND); + } + } + + pr_info("bce_vhci: resume done\n"); + return 0; +} + +static int bce_vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + pr_debug("bce_vhci_urb_enqueue %i:%x\n", q->dev_addr, urb->ep->desc.bEndpointAddress); + if (!q) + return -ENOENT; + return bce_vhci_urb_create(q, urb); +} + +static int bce_vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + pr_debug("bce_vhci_urb_dequeue %x\n", urb->ep->desc.bEndpointAddress); + return bce_vhci_urb_request_cancel(q, urb, status); +} + +static void bce_vhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep) +{ + struct bce_vhci_transfer_queue *q = ep->hcpriv; + pr_debug("bce_vhci_endpoint_reset\n"); + if (q) + bce_vhci_transfer_queue_request_reset(q); +} + +static u8 bce_vhci_endpoint_index(u8 addr) +{ + if (addr & 0x80) + return (u8) (0x10 + (addr & 0xf)); + return (u8) (addr & 0xf); +} + +static int bce_vhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_device *vdev = vhci->devices[devid]; + pr_debug("bce_vhci_add_endpoint %x/%x:%x\n", udev->portnum, devid, endp_index); + + if (udev->bus->root_hub == udev) /* The USB hub */ + return 0; + if (vdev == NULL) + return -ENODEV; + if (vdev->tq_mask & BIT(endp_index)) { + endp->hcpriv = &vdev->tq[endp_index]; + return 0; + } + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[endp_index], endp, devid, + usb_endpoint_dir_in(&endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); + endp->hcpriv = &vdev->tq[endp_index]; + vdev->tq_mask |= BIT(endp_index); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &endp->desc); + return 0; +} + +static int bce_vhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_transfer_queue *q = endp->hcpriv; + struct bce_vhci_device *vdev = vhci->devices[devid]; + pr_info("bce_vhci_drop_endpoint %x:%x\n", udev->portnum, endp_index); + if (!q) { + if (vdev && vdev->tq_mask & BIT(endp_index)) { + pr_err("something deleted the hcpriv?\n"); + q = &vdev->tq[endp_index]; + } else { + return 0; + } + } + + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) (endp->desc.bEndpointAddress & 0x8Fu)); + vhci->devices[devid]->tq_mask &= ~BIT(endp_index); + bce_vhci_destroy_transfer_queue(vhci, q); + return 0; +} + +static int bce_vhci_create_message_queues(struct bce_vhci *vhci) +{ + if (bce_vhci_message_queue_create(vhci, &vhci->msg_commands, "VHC1HostCommands") || + bce_vhci_message_queue_create(vhci, &vhci->msg_system, "VHC1HostSystemEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_isochronous, "VHC1HostIsochronousEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_interrupt, "VHC1HostInterruptEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_asynchronous, "VHC1HostAsynchronousEvents")) { + bce_vhci_destroy_message_queues(vhci); + return -EINVAL; + } + spin_lock_init(&vhci->msg_asynchronous_lock); + bce_vhci_command_queue_create(&vhci->cq, &vhci->msg_commands); + return 0; +} + +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci) +{ + bce_vhci_command_queue_destroy(&vhci->cq); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_commands); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_system); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_isochronous); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_interrupt); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_asynchronous); +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci) +{ + vhci->ev_cq = bce_create_cq(vhci->dev, 0x100); + if (!vhci->ev_cq) + return -EINVAL; +#define CREATE_EVENT_QUEUE(field, name, cb) bce_vhci_event_queue_create(vhci, &vhci->field, name, cb) + if (__bce_vhci_event_queue_create(vhci, &vhci->ev_commands, "VHC1FirmwareCommands", + bce_vhci_firmware_event_completion) || + CREATE_EVENT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", bce_vhci_handle_system_event) || + CREATE_EVENT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", bce_vhci_handle_usb_event)) { + bce_vhci_destroy_event_queues(vhci); + return -EINVAL; + } +#undef CREATE_EVENT_QUEUE + return 0; +} + +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci) +{ + bce_vhci_event_queue_destroy(vhci, &vhci->ev_commands); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_system); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_isochronous); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_interrupt); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_asynchronous); + if (vhci->ev_cq) + bce_destroy_cq(vhci->dev, vhci->ev_cq); +} + +static void bce_vhci_send_fw_event_response(struct bce_vhci *vhci, struct bce_vhci_message *req, u16 status) +{ + unsigned long timeout = 1000; + struct bce_vhci_message r = *req; + r.cmd = (u16) (req->cmd | 0x8000u); + r.status = status; + r.param1 = req->param1; + r.param2 = 0; + + if (bce_reserve_submission(vhci->msg_system.sq, &timeout)) { + pr_err("bce-vhci: Cannot reserve submision for FW event reply\n"); + return; + } + bce_vhci_message_queue_write(&vhci->msg_system, &r); +} + +static int bce_vhci_handle_firmware_event(struct bce_vhci *vhci, struct bce_vhci_message *msg) +{ + unsigned long flags; + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + struct bce_vhci_transfer_queue *tq; + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE || msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = vhci->devices[devid]; + if (!dev || !(dev->tq_mask & BIT(endp))) + return BCE_VHCI_BAD_ARGUMENT; + tq = &dev->tq[endp]; + } + + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_ACTIVE) { + bce_vhci_transfer_queue_resume(tq, BCE_VHCI_PAUSE_FIRMWARE); + return BCE_VHCI_SUCCESS; + } else if (msg->param2 == BCE_VHCI_ENDPOINT_PAUSED) { + bce_vhci_transfer_queue_pause(tq, BCE_VHCI_PAUSE_FIRMWARE); + return BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } else if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_STALLED) { + tq->state = msg->param2; + spin_lock_irqsave(&tq->urb_lock, flags); + tq->stalled = true; + spin_unlock_irqrestore(&tq->urb_lock, flags); + return BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } + pr_warn("bce-vhci: Unhandled firmware event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + return BCE_VHCI_BAD_ARGUMENT; +} + +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws) +{ + size_t cnt = 0; + int result; + struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_fw_events); + struct bce_queue_sq *sq = vhci->ev_commands.sq; + struct bce_sq_completion_data *cq; + struct bce_vhci_message *msg, *msg2 = NULL; + + while (true) { + if (msg2) { + msg = msg2; + msg2 = NULL; + } else if ((cq = bce_next_completion(sq))) { + if (cq->status == BCE_COMPLETION_ABORTED) { + bce_notify_submission_complete(sq); + continue; + } + msg = &vhci->ev_commands.data[sq->head]; + } else { + break; + } + + pr_debug("bce-vhci: Got fw event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + if ((cq = bce_next_completion(sq))) { + msg2 = &vhci->ev_commands.data[(sq->head + 1) % sq->el_count]; + pr_debug("bce-vhci: Got second fw event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + if (cq->status != BCE_COMPLETION_ABORTED && + msg2->cmd == (msg->cmd | 0x4000) && msg2->param1 == msg->param1) { + /* Take two elements */ + pr_debug("bce-vhci: Cancelled\n"); + bce_vhci_send_fw_event_response(vhci, msg, BCE_VHCI_ABORT); + + bce_notify_submission_complete(sq); + bce_notify_submission_complete(sq); + msg2 = NULL; + cnt += 2; + continue; + } + + pr_warn("bce-vhci: Handle fw event - unexpected cancellation\n"); + } + + result = bce_vhci_handle_firmware_event(vhci, msg); + bce_vhci_send_fw_event_response(vhci, msg, (u16) result); + + + bce_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(&vhci->ev_commands, cnt); + if (atomic_read(&sq->available_commands) == sq->el_count - 1) { + pr_debug("bce-vhci: complete\n"); + complete(&vhci->ev_commands.queue_empty_completion); + } +} + +static void bce_vhci_firmware_event_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_event_queue *q = sq->userdata; + queue_work(q->vhci->tq_state_wq, &q->vhci->w_fw_events); +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else { + pr_warn("bce-vhci: Unhandled system event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST || msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = q->vhci->devices[devid]; + if (!dev || (dev->tq_mask & BIT(endp)) == 0) { + pr_err("bce-vhci: Didn't find destination for transfer queue event\n"); + return; + } + bce_vhci_transfer_queue_event(&dev->tq[endp], msg); + } else { + pr_warn("bce-vhci: Unhandled USB event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + + + +static const struct hc_driver bce_vhci_driver = { + .description = "bce-vhci", + .product_desc = "BCE VHCI Host Controller", + .hcd_priv_size = sizeof(struct bce_vhci *), + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + .flags = HCD_USB2, +#else + .flags = HCD_USB2 | HCD_DMA, +#endif + + .start = bce_vhci_start, + .stop = bce_vhci_stop, + .hub_status_data = bce_vhci_hub_status_data, + .hub_control = bce_vhci_hub_control, + .urb_enqueue = bce_vhci_urb_enqueue, + .urb_dequeue = bce_vhci_urb_dequeue, + .enable_device = bce_vhci_enable_device, + .free_dev = bce_vhci_free_device, + .address_device = bce_vhci_address_device, + .add_endpoint = bce_vhci_add_endpoint, + .drop_endpoint = bce_vhci_drop_endpoint, + .endpoint_reset = bce_vhci_endpoint_reset, + .check_bandwidth = bce_vhci_check_bandwidth, + .get_frame_number = bce_vhci_get_frame_number, + .bus_suspend = bce_vhci_bus_suspend, + .bus_resume = bce_vhci_bus_resume +}; + + +int __init bce_vhci_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&bce_vhci_chrdev, 0, 1, "bce-vhci"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_vhci_class = class_create(THIS_MODULE, "bce-vhci"); +#else + bce_vhci_class = class_create("bce-vhci"); +#endif + if (IS_ERR(bce_vhci_class)) { + result = PTR_ERR(bce_vhci_class); + goto fail_class; + } + return 0; + +fail_class: + class_destroy(bce_vhci_class); +fail_chrdev: + unregister_chrdev_region(bce_vhci_chrdev, 1); + if (!result) + result = -EINVAL; + return result; +} +void __exit bce_vhci_module_exit(void) +{ + class_destroy(bce_vhci_class); + unregister_chrdev_region(bce_vhci_chrdev, 1); +} + +module_param_named(vhci_port_mask, bce_vhci_port_mask, ushort, 0444); +MODULE_PARM_DESC(vhci_port_mask, "Specifies which VHCI ports are enabled"); diff --git a/drivers/staging/apple-bce/vhci/vhci.h b/drivers/staging/apple-bce/vhci/vhci.h new file mode 100644 index 000000000000..6c2e22622f4c --- /dev/null +++ b/drivers/staging/apple-bce/vhci/vhci.h @@ -0,0 +1,52 @@ +#ifndef BCE_VHCI_H +#define BCE_VHCI_H + +#include "queue.h" +#include "transfer.h" + +struct usb_hcd; +struct bce_queue_cq; + +struct bce_vhci_device { + struct bce_vhci_transfer_queue tq[32]; + u32 tq_mask; +}; +struct bce_vhci { + struct apple_bce_device *dev; + dev_t vdevt; + struct device *vdev; + struct usb_hcd *hcd; + struct spinlock hcd_spinlock; + struct bce_vhci_message_queue msg_commands; + struct bce_vhci_message_queue msg_system; + struct bce_vhci_message_queue msg_isochronous; + struct bce_vhci_message_queue msg_interrupt; + struct bce_vhci_message_queue msg_asynchronous; + struct spinlock msg_asynchronous_lock; + struct bce_vhci_command_queue cq; + struct bce_queue_cq *ev_cq; + struct bce_vhci_event_queue ev_commands; + struct bce_vhci_event_queue ev_system; + struct bce_vhci_event_queue ev_isochronous; + struct bce_vhci_event_queue ev_interrupt; + struct bce_vhci_event_queue ev_asynchronous; + u16 port_mask; + u8 port_count; + u16 port_power_mask; + bce_vhci_device_t port_to_device[16]; + struct bce_vhci_device *devices[16]; + struct workqueue_struct *tq_state_wq; + struct work_struct w_fw_events; +}; + +int __init bce_vhci_module_init(void); +void __exit bce_vhci_module_exit(void); + +int bce_vhci_create(struct apple_bce_device *dev, struct bce_vhci *vhci); +void bce_vhci_destroy(struct bce_vhci *vhci); +int bce_vhci_start(struct usb_hcd *hcd); +void bce_vhci_stop(struct usb_hcd *hcd); + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd); + +#endif //BCE_VHCI_H diff --git a/include/linux/hid.h b/include/linux/hid.h index 7f260e0e2049..0075b9956d05 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -594,7 +594,9 @@ struct hid_input { enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -755,6 +757,8 @@ struct hid_descriptor { .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h new file mode 100644 index 000000000000..0b7093935ddf --- /dev/null +++ b/include/linux/soc/apple/dockchannel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif -- 2.51.0