From 6a4c01f2a56b28d16dd65e2c3470e3c65cbdb4ab Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:54:45 +0200 Subject: [PATCH 1/8] amd-pstate Signed-off-by: Peter Jung --- drivers/cpufreq/amd-pstate.c | 111 +++++++++++++++++++++++++-------- drivers/cpufreq/amd-pstate.h | 2 + include/linux/sched/topology.h | 6 ++ kernel/sched/debug.c | 4 ++ kernel/sched/fair.c | 5 +- kernel/sched/topology.c | 58 +++++++++++++++++ 6 files changed, 158 insertions(+), 28 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b961f3a3b580..12331e127d96 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -389,7 +389,8 @@ static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) static int msr_init_perf(struct amd_cpudata *cpudata) { union perf_cached perf = READ_ONCE(cpudata->perf); - u64 cap1, numerator; + u64 cap1, numerator, cppc_req; + u8 min_perf; int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); @@ -400,6 +401,22 @@ static int msr_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); + if (ret) + return ret; + + WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); + min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); + + /* + * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an + * indication that the min_perf value is the one specified through the BIOS option + */ + cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); + + if (!cppc_req) + perf.bios_min_perf = min_perf; + perf.highest_perf = numerator; perf.max_limit_perf = numerator; perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); @@ -554,6 +571,10 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, if (!policy) return; + /* limit the max perf when core performance boost feature is disabled */ + if (!cpudata->boost_supported) + max_perf = min_t(u8, perf.nominal_perf, max_perf); + des_perf = clamp_t(u8, des_perf, min_perf, max_perf); policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); @@ -563,10 +584,6 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, des_perf = 0; } - /* limit the max perf when core performance boost feature is disabled */ - if (!cpudata->boost_supported) - max_perf = min_t(u8, perf.nominal_perf, max_perf); - if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, @@ -580,20 +597,26 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) { /* * Initialize lower frequency limit (i.e.policy->min) with - * lowest_nonlinear_frequency which is the most energy efficient - * frequency. Override the initial value set by cpufreq core and - * amd-pstate qos_requests. + * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, + * Override the initial value set by cpufreq core and amd-pstate qos_requests. */ if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(policy_data->cpu); struct amd_cpudata *cpudata; + union perf_cached perf; if (!policy) return -EINVAL; cpudata = policy->driver_data; - policy_data->min = cpudata->lowest_nonlinear_freq; + perf = READ_ONCE(cpudata->perf); + + if (perf.bios_min_perf) + policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, + perf.bios_min_perf); + else + policy_data->min = cpudata->lowest_nonlinear_freq; } cpufreq_verify_within_cpu_limits(policy_data); @@ -831,8 +854,10 @@ static void amd_pstate_update_limits(unsigned int cpu) if (highest_perf_changed) { WRITE_ONCE(cpudata->prefcore_ranking, cur_high); - if (cur_high < CPPC_MAX_PERF) + if (cur_high < CPPC_MAX_PERF) { sched_set_itmt_core_prio((int)cur_high, cpu); + sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); + } } } @@ -1024,6 +1049,10 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* Reset CPPC_REQ MSR to the BIOS value */ + amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); freq_qos_remove_request(&cpudata->req[1]); freq_qos_remove_request(&cpudata->req[0]); @@ -1419,7 +1448,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) struct amd_cpudata *cpudata; union perf_cached perf; struct device *dev; - u64 value; int ret; /* @@ -1484,12 +1512,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; } - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); - if (ret) - return ret; - WRITE_ONCE(cpudata->cppc_req_cached, value); - } ret = amd_pstate_set_epp(policy, cpudata->epp_default); if (ret) return ret; @@ -1509,6 +1531,11 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; if (cpudata) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* Reset CPPC_REQ MSR to the BIOS value */ + amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + kfree(cpudata); policy->driver_data = NULL; } @@ -1559,21 +1586,38 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) return 0; } -static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) +static int amd_pstate_cpu_online(struct cpufreq_policy *policy) { - pr_debug("AMD CPU Core %d going online\n", policy->cpu); - return amd_pstate_cppc_enable(policy); } -static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) +static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) { - return 0; + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* + * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified + * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the + * limits, epp and desired perf will get reset to the cached values in cpudata struct + */ + return amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); } -static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) +static int amd_pstate_suspend(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + int ret; + + /* + * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified + * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, + * the limits, epp and desired perf will get reset to the cached values in cpudata struct + */ + ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + if (ret) + return ret; /* invalidate to ensure it's rewritten during resume */ cpudata->cppc_req_cached = 0; @@ -1584,6 +1628,17 @@ static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) return 0; } +static int amd_pstate_resume(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); + + /* Set CPPC_REQ to last sane value until the governor updates it */ + return amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, + 0U, false); +} + static int amd_pstate_epp_resume(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1609,6 +1664,10 @@ static struct cpufreq_driver amd_pstate_driver = { .fast_switch = amd_pstate_fast_switch, .init = amd_pstate_cpu_init, .exit = amd_pstate_cpu_exit, + .online = amd_pstate_cpu_online, + .offline = amd_pstate_cpu_offline, + .suspend = amd_pstate_suspend, + .resume = amd_pstate_resume, .set_boost = amd_pstate_set_boost, .update_limits = amd_pstate_update_limits, .name = "amd-pstate", @@ -1621,9 +1680,9 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .setpolicy = amd_pstate_epp_set_policy, .init = amd_pstate_epp_cpu_init, .exit = amd_pstate_epp_cpu_exit, - .offline = amd_pstate_epp_cpu_offline, - .online = amd_pstate_epp_cpu_online, - .suspend = amd_pstate_epp_suspend, + .offline = amd_pstate_cpu_offline, + .online = amd_pstate_cpu_online, + .suspend = amd_pstate_suspend, .resume = amd_pstate_epp_resume, .update_limits = amd_pstate_update_limits, .set_boost = amd_pstate_set_boost, diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index fbe1c08d3f06..2f7ae364d331 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -30,6 +30,7 @@ * @lowest_perf: the absolute lowest performance level of the processor * @min_limit_perf: Cached value of the performance corresponding to policy->min * @max_limit_perf: Cached value of the performance corresponding to policy->max + * @bios_min_perf: Cached perf value corresponding to the "Requested CPU Min Frequency" BIOS option */ union perf_cached { struct { @@ -39,6 +40,7 @@ union perf_cached { u8 lowest_perf; u8 min_limit_perf; u8 max_limit_perf; + u8 bios_min_perf; }; u64 val; }; diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 7b4301b7235f..198bb5cc1774 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -195,6 +195,8 @@ struct sched_domain_topology_level { }; extern void __init set_sched_topology(struct sched_domain_topology_level *tl); +extern void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio); + # define SD_INIT_NAME(type) .name = #type @@ -223,6 +225,10 @@ static inline bool cpus_share_resources(int this_cpu, int that_cpu) return true; } +static inline void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio) +{ +} + #endif /* !CONFIG_SMP */ #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 56ae54e0ce6a..557246880a7e 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -588,6 +588,10 @@ static void register_sd(struct sched_domain *sd, struct dentry *parent) debugfs_create_file("flags", 0444, parent, &sd->flags, &sd_flags_fops); debugfs_create_file("groups_flags", 0444, parent, &sd->groups->flags, &sd_flags_fops); debugfs_create_u32("level", 0444, parent, (u32 *)&sd->level); + + if (sd->flags & SD_ASYM_PACKING) + debugfs_create_u32("group_asym_prefer_cpu", 0444, parent, + (u32 *)&sd->groups->asym_prefer_cpu); } void update_sched_domain_debugfs(void) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0fb9bf995a47..8d0f462e8c8b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10256,7 +10256,7 @@ sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group (sgs->group_weight - sgs->idle_cpus != 1)) return false; - return sched_asym(env->sd, env->dst_cpu, group->asym_prefer_cpu); + return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu)); } /* One group has more than one SMT CPU while the other group does not */ @@ -10493,7 +10493,8 @@ static bool update_sd_pick_busiest(struct lb_env *env, case group_asym_packing: /* Prefer to move from lowest priority CPU's work */ - return sched_asym_prefer(sds->busiest->asym_prefer_cpu, sg->asym_prefer_cpu); + return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu), + READ_ONCE(sg->asym_prefer_cpu)); case group_misfit_task: /* diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index f1ebc60d967f..8426de317835 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1333,6 +1333,64 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) update_group_capacity(sd, cpu); } +#ifdef CONFIG_SMP + +/* Update the "asym_prefer_cpu" when arch_asym_cpu_priority() changes. */ +void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio) +{ + int asym_prefer_cpu = cpu; + struct sched_domain *sd; + + guard(rcu)(); + + for_each_domain(cpu, sd) { + struct sched_group *sg; + int group_cpu; + + if (!(sd->flags & SD_ASYM_PACKING)) + continue; + + /* + * Groups of overlapping domain are replicated per NUMA + * node and will require updating "asym_prefer_cpu" on + * each local copy. + * + * If you are hitting this warning, consider moving + * "sg->asym_prefer_cpu" to "sg->sgc->asym_prefer_cpu" + * which is shared by all the overlapping groups. + */ + WARN_ON_ONCE(sd->flags & SD_OVERLAP); + + sg = sd->groups; + if (cpu != sg->asym_prefer_cpu) { + /* + * Since the parent is a superset of the current group, + * if the cpu is not the "asym_prefer_cpu" at the + * current level, it cannot be the preferred CPU at a + * higher levels either. + */ + if (!sched_asym_prefer(cpu, sg->asym_prefer_cpu)) + return; + + WRITE_ONCE(sg->asym_prefer_cpu, cpu); + continue; + } + + /* Ranking has improved; CPU is still the preferred one. */ + if (new_prio >= old_prio) + continue; + + for_each_cpu(group_cpu, sched_group_span(sg)) { + if (sched_asym_prefer(group_cpu, asym_prefer_cpu)) + asym_prefer_cpu = group_cpu; + } + + WRITE_ONCE(sg->asym_prefer_cpu, asym_prefer_cpu); + } +} + +#endif /* CONFIG_SMP */ + /* * Set of available CPUs grouped by their corresponding capacities * Each list entry contains a CPU mask reflecting CPUs that share the same -- 2.49.0.654.g845c48a16a From b74d970a4d8fa6a0c70daaf77abbb3846aff2126 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:54:55 +0200 Subject: [PATCH 2/8] asus Signed-off-by: Peter Jung --- .../ABI/testing/sysfs-platform-asus-wmi | 17 + .../display/dc/dml2/dml2_translation_helper.c | 2 +- drivers/hid/Kconfig | 2 + drivers/hid/Makefile | 2 + drivers/hid/asus-ally-hid/Kconfig | 8 + drivers/hid/asus-ally-hid/Makefile | 6 + .../hid/asus-ally-hid/asus-ally-hid-config.c | 2347 +++++++++++++++++ .../hid/asus-ally-hid/asus-ally-hid-core.c | 600 +++++ .../hid/asus-ally-hid/asus-ally-hid-input.c | 345 +++ drivers/hid/asus-ally-hid/asus-ally-rgb.c | 356 +++ drivers/hid/asus-ally-hid/asus-ally.h | 314 +++ drivers/hid/hid-asus.c | 116 +- drivers/hid/hid-ids.h | 1 + drivers/platform/x86/Kconfig | 23 + drivers/platform/x86/Makefile | 1 + drivers/platform/x86/asus-armoury.c | 1202 +++++++++ drivers/platform/x86/asus-armoury.h | 1278 +++++++++ drivers/platform/x86/asus-wmi.c | 359 ++- include/linux/platform_data/x86/asus-wmi.h | 43 + 19 files changed, 6915 insertions(+), 107 deletions(-) create mode 100644 drivers/hid/asus-ally-hid/Kconfig create mode 100644 drivers/hid/asus-ally-hid/Makefile create mode 100644 drivers/hid/asus-ally-hid/asus-ally-hid-config.c create mode 100644 drivers/hid/asus-ally-hid/asus-ally-hid-core.c create mode 100644 drivers/hid/asus-ally-hid/asus-ally-hid-input.c create mode 100644 drivers/hid/asus-ally-hid/asus-ally-rgb.c create mode 100644 drivers/hid/asus-ally-hid/asus-ally.h create mode 100644 drivers/platform/x86/asus-armoury.c create mode 100644 drivers/platform/x86/asus-armoury.h diff --git a/Documentation/ABI/testing/sysfs-platform-asus-wmi b/Documentation/ABI/testing/sysfs-platform-asus-wmi index 28144371a0f1..765d50b0d9df 100644 --- a/Documentation/ABI/testing/sysfs-platform-asus-wmi +++ b/Documentation/ABI/testing/sysfs-platform-asus-wmi @@ -63,6 +63,7 @@ Date: Aug 2022 KernelVersion: 6.1 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Switch the GPU hardware MUX mode. Laptops with this feature can can be toggled to boot with only the dGPU (discrete mode) or in standard Optimus/Hybrid mode. On switch a reboot is required: @@ -75,6 +76,7 @@ Date: Aug 2022 KernelVersion: 5.17 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Disable discrete GPU: * 0 - Enable dGPU, * 1 - Disable dGPU @@ -84,6 +86,7 @@ Date: Aug 2022 KernelVersion: 5.17 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Enable the external GPU paired with ROG X-Flow laptops. Toggling this setting will also trigger ACPI to disable the dGPU: @@ -95,6 +98,7 @@ Date: Aug 2022 KernelVersion: 5.17 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Enable an LCD response-time boost to reduce or remove ghosting: * 0 - Disable, * 1 - Enable @@ -104,6 +108,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Get the current charging mode being used: * 1 - Barrel connected charger, * 2 - USB-C charging @@ -114,6 +119,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Show if the egpu (XG Mobile) is correctly connected: * 0 - False, * 1 - True @@ -123,6 +129,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Change the mini-LED mode: * 0 - Single-zone, * 1 - Multi-zone @@ -133,6 +140,7 @@ Date: Apr 2024 KernelVersion: 6.10 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON List the available mini-led modes. What: /sys/devices/platform//ppt_pl1_spl @@ -140,6 +148,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the Package Power Target total of CPU: PL1 on Intel, SPL on AMD. Shown on Intel+Nvidia or AMD+Nvidia based systems: @@ -150,6 +159,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the Slow Package Power Tracking Limit of CPU: PL2 on Intel, SPPT, on AMD. Shown on Intel+Nvidia or AMD+Nvidia based systems: @@ -160,6 +170,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the Fast Package Power Tracking Limit of CPU. AMD+Nvidia only: * min=5, max=250 @@ -168,6 +179,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the APU SPPT limit. Shown on full AMD systems only: * min=5, max=130 @@ -176,6 +188,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the platform SPPT limit. Shown on full AMD systems only: * min=5, max=130 @@ -184,6 +197,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the dynamic boost limit of the Nvidia dGPU: * min=5, max=25 @@ -192,6 +206,7 @@ Date: Jun 2023 KernelVersion: 6.5 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set the target temperature limit of the Nvidia dGPU: * min=75, max=87 @@ -200,6 +215,7 @@ Date: Apr 2024 KernelVersion: 6.10 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set if the BIOS POST sound is played on boot. * 0 - False, * 1 - True @@ -209,6 +225,7 @@ Date: Apr 2024 KernelVersion: 6.10 Contact: "Luke Jones" Description: + DEPRECATED, WILL BE REMOVED SOON Set if the MCU can go in to low-power mode on system sleep * 0 - False, * 1 - True diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c index ab6baf269801..5de775fd8fce 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c @@ -896,7 +896,7 @@ static void populate_dummy_dml_surface_cfg(struct dml_surface_cfg_st *out, unsig out->SurfaceWidthC[location] = in->timing.h_addressable; out->SurfaceHeightC[location] = in->timing.v_addressable; out->PitchY[location] = ((out->SurfaceWidthY[location] + 127) / 128) * 128; - out->PitchC[location] = 0; + out->PitchC[location] = 1; out->DCCEnable[location] = false; out->DCCMetaPitchY[location] = 0; out->DCCMetaPitchC[location] = 0; diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index a503252702b7..f83b415092ba 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -1423,6 +1423,8 @@ source "drivers/hid/intel-ish-hid/Kconfig" source "drivers/hid/amd-sfh-hid/Kconfig" +source "drivers/hid/asus-ally-hid/Kconfig" + source "drivers/hid/surface-hid/Kconfig" source "drivers/hid/intel-thc-hid/Kconfig" diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 10ae5dedbd84..85af5331ebd2 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -172,6 +172,8 @@ obj-$(CONFIG_INTEL_ISH_HID) += intel-ish-hid/ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ +obj-$(CONFIG_ASUS_ALLY_HID) += asus-ally-hid/ + obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ diff --git a/drivers/hid/asus-ally-hid/Kconfig b/drivers/hid/asus-ally-hid/Kconfig new file mode 100644 index 000000000000..f83dda32be62 --- /dev/null +++ b/drivers/hid/asus-ally-hid/Kconfig @@ -0,0 +1,8 @@ +config ASUS_ALLY_HID + tristate "Asus Ally handheld support" + depends on USB_HID + depends on LEDS_CLASS + depends on LEDS_CLASS_MULTICOLOR + select POWER_SUPPLY + help + Support for configuring the Asus ROG Ally gamepad using attributes. diff --git a/drivers/hid/asus-ally-hid/Makefile b/drivers/hid/asus-ally-hid/Makefile new file mode 100644 index 000000000000..5c3c304b7b53 --- /dev/null +++ b/drivers/hid/asus-ally-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Makefile - ASUS ROG Ally handheld device driver +# +asus-ally-hid-y := asus-ally-hid-core.o asus-ally-rgb.o asus-ally-hid-input.o asus-ally-hid-config.o +obj-$(CONFIG_ASUS_ALLY_HID) := asus-ally-hid.o diff --git a/drivers/hid/asus-ally-hid/asus-ally-hid-config.c b/drivers/hid/asus-ally-hid/asus-ally-hid-config.c new file mode 100644 index 000000000000..1624880121c4 --- /dev/null +++ b/drivers/hid/asus-ally-hid/asus-ally-hid-config.c @@ -0,0 +1,2347 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2023 Luke Jones + */ + +#include +#include +#include +#include +#include +#include + +#include "asus-ally.h" +#include "../hid-ids.h" + +enum btn_map_type { + BTN_TYPE_NONE = 0, + BTN_TYPE_PAD = 0x01, + BTN_TYPE_KB = 0x02, + BTN_TYPE_MOUSE = 0x03, + BTN_TYPE_MEDIA = 0x05, +}; + +struct btn_code_map { + unsigned char type; + unsigned char value; + const char *name; +}; + +static const struct btn_code_map ally_btn_codes[] = { + { BTN_TYPE_NONE, 0x00, "NONE" }, + /* Gamepad button codes */ + { BTN_TYPE_PAD, 0x01, "PAD_A" }, + { BTN_TYPE_PAD, 0x02, "PAD_B" }, + { BTN_TYPE_PAD, 0x03, "PAD_X" }, + { BTN_TYPE_PAD, 0x04, "PAD_Y" }, + { BTN_TYPE_PAD, 0x05, "PAD_LB" }, + { BTN_TYPE_PAD, 0x06, "PAD_RB" }, + { BTN_TYPE_PAD, 0x07, "PAD_LS" }, + { BTN_TYPE_PAD, 0x08, "PAD_RS" }, + { BTN_TYPE_PAD, 0x09, "PAD_DPAD_UP" }, + { BTN_TYPE_PAD, 0x0A, "PAD_DPAD_DOWN" }, + { BTN_TYPE_PAD, 0x0B, "PAD_DPAD_LEFT" }, + { BTN_TYPE_PAD, 0x0C, "PAD_DPAD_RIGHT" }, + { BTN_TYPE_PAD, 0x0D, "PAD_LT" }, + { BTN_TYPE_PAD, 0x0E, "PAD_RT" }, + { BTN_TYPE_PAD, 0x11, "PAD_VIEW" }, + { BTN_TYPE_PAD, 0x12, "PAD_MENU" }, + { BTN_TYPE_PAD, 0x13, "PAD_XBOX" }, + + /* Keyboard button codes */ + { BTN_TYPE_KB, 0x8E, "KB_M2" }, + { BTN_TYPE_KB, 0x8F, "KB_M1" }, + { BTN_TYPE_KB, 0x76, "KB_ESC" }, + { BTN_TYPE_KB, 0x50, "KB_F1" }, + { BTN_TYPE_KB, 0x60, "KB_F2" }, + { BTN_TYPE_KB, 0x40, "KB_F3" }, + { BTN_TYPE_KB, 0x0C, "KB_F4" }, + { BTN_TYPE_KB, 0x03, "KB_F5" }, + { BTN_TYPE_KB, 0x0B, "KB_F6" }, + { BTN_TYPE_KB, 0x80, "KB_F7" }, + { BTN_TYPE_KB, 0x0A, "KB_F8" }, + { BTN_TYPE_KB, 0x01, "KB_F9" }, + { BTN_TYPE_KB, 0x09, "KB_F10" }, + { BTN_TYPE_KB, 0x78, "KB_F11" }, + { BTN_TYPE_KB, 0x07, "KB_F12" }, + { BTN_TYPE_KB, 0x18, "KB_F14" }, + { BTN_TYPE_KB, 0x10, "KB_F15" }, + { BTN_TYPE_KB, 0x0E, "KB_BACKTICK" }, + { BTN_TYPE_KB, 0x16, "KB_1" }, + { BTN_TYPE_KB, 0x1E, "KB_2" }, + { BTN_TYPE_KB, 0x26, "KB_3" }, + { BTN_TYPE_KB, 0x25, "KB_4" }, + { BTN_TYPE_KB, 0x2E, "KB_5" }, + { BTN_TYPE_KB, 0x36, "KB_6" }, + { BTN_TYPE_KB, 0x3D, "KB_7" }, + { BTN_TYPE_KB, 0x3E, "KB_8" }, + { BTN_TYPE_KB, 0x46, "KB_9" }, + { BTN_TYPE_KB, 0x45, "KB_0" }, + { BTN_TYPE_KB, 0x4E, "KB_HYPHEN" }, + { BTN_TYPE_KB, 0x55, "KB_EQUALS" }, + { BTN_TYPE_KB, 0x66, "KB_BACKSPACE" }, + { BTN_TYPE_KB, 0x0D, "KB_TAB" }, + { BTN_TYPE_KB, 0x15, "KB_Q" }, + { BTN_TYPE_KB, 0x1D, "KB_W" }, + { BTN_TYPE_KB, 0x24, "KB_E" }, + { BTN_TYPE_KB, 0x2D, "KB_R" }, + { BTN_TYPE_KB, 0x2C, "KB_T" }, + { BTN_TYPE_KB, 0x35, "KB_Y" }, + { BTN_TYPE_KB, 0x3C, "KB_U" }, + { BTN_TYPE_KB, 0x44, "KB_O" }, + { BTN_TYPE_KB, 0x4D, "KB_P" }, + { BTN_TYPE_KB, 0x54, "KB_LBRACKET" }, + { BTN_TYPE_KB, 0x5B, "KB_RBRACKET" }, + { BTN_TYPE_KB, 0x5D, "KB_BACKSLASH" }, + { BTN_TYPE_KB, 0x58, "KB_CAPS" }, + { BTN_TYPE_KB, 0x1C, "KB_A" }, + { BTN_TYPE_KB, 0x1B, "KB_S" }, + { BTN_TYPE_KB, 0x23, "KB_D" }, + { BTN_TYPE_KB, 0x2B, "KB_F" }, + { BTN_TYPE_KB, 0x34, "KB_G" }, + { BTN_TYPE_KB, 0x33, "KB_H" }, + { BTN_TYPE_KB, 0x3B, "KB_J" }, + { BTN_TYPE_KB, 0x42, "KB_K" }, + { BTN_TYPE_KB, 0x4B, "KB_L" }, + { BTN_TYPE_KB, 0x4C, "KB_SEMI" }, + { BTN_TYPE_KB, 0x52, "KB_QUOTE" }, + { BTN_TYPE_KB, 0x5A, "KB_RET" }, + { BTN_TYPE_KB, 0x88, "KB_LSHIFT" }, + { BTN_TYPE_KB, 0x1A, "KB_Z" }, + { BTN_TYPE_KB, 0x22, "KB_X" }, + { BTN_TYPE_KB, 0x21, "KB_C" }, + { BTN_TYPE_KB, 0x2A, "KB_V" }, + { BTN_TYPE_KB, 0x32, "KB_B" }, + { BTN_TYPE_KB, 0x31, "KB_N" }, + { BTN_TYPE_KB, 0x3A, "KB_M" }, + { BTN_TYPE_KB, 0x41, "KB_COMMA" }, + { BTN_TYPE_KB, 0x49, "KB_PERIOD" }, + { BTN_TYPE_KB, 0x89, "KB_RSHIFT" }, + { BTN_TYPE_KB, 0x8C, "KB_LCTL" }, + { BTN_TYPE_KB, 0x82, "KB_META" }, + { BTN_TYPE_KB, 0x8A, "KB_LALT" }, + { BTN_TYPE_KB, 0x29, "KB_SPACE" }, + { BTN_TYPE_KB, 0x8B, "KB_RALT" }, + { BTN_TYPE_KB, 0x84, "KB_MENU" }, + { BTN_TYPE_KB, 0x8D, "KB_RCTL" }, + { BTN_TYPE_KB, 0xC3, "KB_PRNTSCN" }, + { BTN_TYPE_KB, 0x7E, "KB_SCRLCK" }, + { BTN_TYPE_KB, 0x91, "KB_PAUSE" }, + { BTN_TYPE_KB, 0xC2, "KB_INS" }, + { BTN_TYPE_KB, 0x94, "KB_HOME" }, + { BTN_TYPE_KB, 0x96, "KB_PGUP" }, + { BTN_TYPE_KB, 0xC0, "KB_DEL" }, + { BTN_TYPE_KB, 0x95, "KB_END" }, + { BTN_TYPE_KB, 0x97, "KB_PGDWN" }, + { BTN_TYPE_KB, 0x98, "KB_UP_ARROW" }, + { BTN_TYPE_KB, 0x99, "KB_DOWN_ARROW" }, + { BTN_TYPE_KB, 0x91, "KB_LEFT_ARROW" }, + { BTN_TYPE_KB, 0x9B, "KB_RIGHT_ARROW" }, + + /* Numpad button codes */ + { BTN_TYPE_KB, 0x77, "NUMPAD_LOCK" }, + { BTN_TYPE_KB, 0x90, "NUMPAD_FWDSLASH" }, + { BTN_TYPE_KB, 0x7C, "NUMPAD_ASTERISK" }, + { BTN_TYPE_KB, 0x7B, "NUMPAD_HYPHEN" }, + { BTN_TYPE_KB, 0x70, "NUMPAD_0" }, + { BTN_TYPE_KB, 0x69, "NUMPAD_1" }, + { BTN_TYPE_KB, 0x72, "NUMPAD_2" }, + { BTN_TYPE_KB, 0x7A, "NUMPAD_3" }, + { BTN_TYPE_KB, 0x6B, "NUMPAD_4" }, + { BTN_TYPE_KB, 0x73, "NUMPAD_5" }, + { BTN_TYPE_KB, 0x74, "NUMPAD_6" }, + { BTN_TYPE_KB, 0x6C, "NUMPAD_7" }, + { BTN_TYPE_KB, 0x75, "NUMPAD_8" }, + { BTN_TYPE_KB, 0x7D, "NUMPAD_9" }, + { BTN_TYPE_KB, 0x79, "NUMPAD_PLUS" }, + { BTN_TYPE_KB, 0x81, "NUMPAD_ENTER" }, + { BTN_TYPE_KB, 0x71, "NUMPAD_PERIOD" }, + + /* Mouse button codes */ + { BTN_TYPE_MOUSE, 0x01, "MOUSE_LCLICK" }, + { BTN_TYPE_MOUSE, 0x02, "MOUSE_RCLICK" }, + { BTN_TYPE_MOUSE, 0x03, "MOUSE_MCLICK" }, + { BTN_TYPE_MOUSE, 0x04, "MOUSE_WHEEL_UP" }, + { BTN_TYPE_MOUSE, 0x05, "MOUSE_WHEEL_DOWN" }, + + /* Media button codes */ + { BTN_TYPE_MEDIA, 0x16, "MEDIA_SCREENSHOT" }, + { BTN_TYPE_MEDIA, 0x19, "MEDIA_SHOW_KEYBOARD" }, + { BTN_TYPE_MEDIA, 0x1C, "MEDIA_SHOW_DESKTOP" }, + { BTN_TYPE_MEDIA, 0x1E, "MEDIA_START_RECORDING" }, + { BTN_TYPE_MEDIA, 0x01, "MEDIA_MIC_OFF" }, + { BTN_TYPE_MEDIA, 0x02, "MEDIA_VOL_DOWN" }, + { BTN_TYPE_MEDIA, 0x03, "MEDIA_VOL_UP" }, +}; + +static const size_t keymap_len = ARRAY_SIZE(ally_btn_codes); + +/* Button pair indexes for mapping commands */ +enum btn_pair_index { + BTN_PAIR_DPAD_UPDOWN = 0x01, + BTN_PAIR_DPAD_LEFTRIGHT = 0x02, + BTN_PAIR_STICK_LR = 0x03, + BTN_PAIR_BUMPER_LR = 0x04, + BTN_PAIR_AB = 0x05, + BTN_PAIR_XY = 0x06, + BTN_PAIR_VIEW_MENU = 0x07, + BTN_PAIR_M1M2 = 0x08, + BTN_PAIR_TRIGGER_LR = 0x09, +}; + +struct button_map { + struct btn_code_map *remap; + struct btn_code_map *macro; +}; + +struct button_pair_map { + enum btn_pair_index pair_index; + struct button_map first; + struct button_map second; +}; + +/* Store button mapping per gamepad mode */ +struct ally_button_mapping { + struct button_pair_map button_pairs[9]; /* 9 button pairs */ +}; + +/* Find a button code map by its name */ +static const struct btn_code_map *find_button_by_name(const char *name) +{ + int i; + + for (i = 0; i < keymap_len; i++) { + if (strcmp(ally_btn_codes[i].name, name) == 0) + return &ally_btn_codes[i]; + } + + return NULL; +} + +/* Set button mapping for a button pair */ +static int ally_set_button_mapping(struct hid_device *hdev, struct ally_handheld *ally, + struct button_pair_map *mapping) +{ + unsigned char packet[64] = { 0 }; + + if (!mapping) + return -EINVAL; + + packet[0] = HID_ALLY_SET_REPORT_ID; + packet[1] = HID_ALLY_FEATURE_CODE_PAGE; + packet[2] = CMD_SET_MAPPING; + packet[3] = mapping->pair_index; + packet[4] = 0x2C; /* Length */ + + /* First button mapping */ + packet[5] = mapping->first.remap->type; + /* Fill in bytes 6-14 with button code */ + if (mapping->first.remap->type) { + unsigned char btn_bytes[10] = {0}; + btn_bytes[0] = mapping->first.remap->type; + + switch (mapping->first.remap->type) { + case BTN_TYPE_NONE: + break; + case BTN_TYPE_PAD: + case BTN_TYPE_KB: + case BTN_TYPE_MEDIA: + btn_bytes[2] = mapping->first.remap->value; + break; + case BTN_TYPE_MOUSE: + btn_bytes[4] = mapping->first.remap->value; + break; + } + memcpy(&packet[5], btn_bytes, 10); + } + + /* Macro mapping for first button if any */ + packet[15] = mapping->first.macro->type; + if (mapping->first.macro->type) { + unsigned char macro_bytes[11] = {0}; + macro_bytes[0] = mapping->first.macro->type; + + switch (mapping->first.macro->type) { + case BTN_TYPE_NONE: + break; + case BTN_TYPE_PAD: + case BTN_TYPE_KB: + case BTN_TYPE_MEDIA: + macro_bytes[2] = mapping->first.macro->value; + break; + case BTN_TYPE_MOUSE: + macro_bytes[4] = mapping->first.macro->value; + break; + } + memcpy(&packet[15], macro_bytes, 11); + } + + /* Second button mapping */ + packet[27] = mapping->second.remap->type; + /* Fill in bytes 28-36 with button code */ + if (mapping->second.remap->type) { + unsigned char btn_bytes[10] = {0}; + btn_bytes[0] = mapping->second.remap->type; + + switch (mapping->second.remap->type) { + case BTN_TYPE_NONE: + break; + case BTN_TYPE_PAD: + case BTN_TYPE_KB: + case BTN_TYPE_MEDIA: + btn_bytes[2] = mapping->second.remap->value; + break; + case BTN_TYPE_MOUSE: + btn_bytes[4] = mapping->second.remap->value; + break; + } + memcpy(&packet[27], btn_bytes, 10); + } + + /* Macro mapping for second button if any */ + packet[37] = mapping->second.macro->type; + if (mapping->second.macro->type) { + unsigned char macro_bytes[11] = {0}; + macro_bytes[0] = mapping->second.macro->type; + + switch (mapping->second.macro->type) { + case BTN_TYPE_NONE: + break; + case BTN_TYPE_PAD: + case BTN_TYPE_KB: + case BTN_TYPE_MEDIA: + macro_bytes[2] = mapping->second.macro->value; + break; + case BTN_TYPE_MOUSE: + macro_bytes[4] = mapping->second.macro->value; + break; + } + memcpy(&packet[37], macro_bytes, 11); + } + + return ally_gamepad_send_packet(ally, hdev, packet, sizeof(packet)); +} + +/** + * ally_check_capability - Check if a specific capability is supported + * @hdev: HID device + * @flag_code: Capability flag code to check + * + * Returns true if capability is supported, false otherwise + */ +static bool ally_check_capability(struct hid_device *hdev, u8 flag_code) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + bool result = false; + u8 *hidbuf; + int ret; + + hidbuf = kzalloc(HID_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return false; + + hidbuf[0] = HID_ALLY_SET_REPORT_ID; + hidbuf[1] = HID_ALLY_FEATURE_CODE_PAGE; + hidbuf[2] = flag_code; + hidbuf[3] = 0x01; + + ret = ally_gamepad_send_receive_packet(ally, hdev, hidbuf, HID_ALLY_REPORT_SIZE); + if (ret < 0) + goto cleanup; + + if (hidbuf[1] == HID_ALLY_FEATURE_CODE_PAGE && hidbuf[2] == flag_code) + result = (hidbuf[4] == 0x01); + +cleanup: + kfree(hidbuf); + return result; +} + +static int ally_detect_capabilities(struct hid_device *hdev, + struct ally_config *cfg) +{ + if (!hdev || !cfg) + return -EINVAL; + + mutex_lock(&cfg->config_mutex); + cfg->is_ally_x = + (hdev->product == USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X); + + cfg->xbox_controller_support = + ally_check_capability(hdev, CMD_CHECK_XBOX_SUPPORT); + cfg->user_cal_support = + ally_check_capability(hdev, CMD_CHECK_USER_CAL_SUPPORT); + cfg->turbo_support = + ally_check_capability(hdev, CMD_CHECK_TURBO_SUPPORT); + cfg->resp_curve_support = + ally_check_capability(hdev, CMD_CHECK_RESP_CURVE_SUPPORT); + cfg->dir_to_btn_support = + ally_check_capability(hdev, CMD_CHECK_DIR_TO_BTN_SUPPORT); + cfg->gyro_support = + ally_check_capability(hdev, CMD_CHECK_GYRO_TO_JOYSTICK); + cfg->anti_deadzone_support = + ally_check_capability(hdev, CMD_CHECK_ANTI_DEADZONE); + mutex_unlock(&cfg->config_mutex); + + hid_dbg( + hdev, + "Ally capabilities: %s, Xbox: %d, UserCal: %d, Turbo: %d, RespCurve: %d, DirToBtn: %d, Gyro: %d, AntiDZ: %d", + cfg->is_ally_x ? "Ally X" : "Ally", + cfg->xbox_controller_support, cfg->user_cal_support, + cfg->turbo_support, cfg->resp_curve_support, + cfg->dir_to_btn_support, cfg->gyro_support, + cfg->anti_deadzone_support); + + return 0; +} + +static int ally_set_xbox_controller(struct hid_device *hdev, + struct ally_config *cfg, bool enabled) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + u8 buffer[64] = { 0 }; + int ret; + + if (!cfg || !cfg->xbox_controller_support) + return -ENODEV; + + buffer[0] = HID_ALLY_SET_REPORT_ID; + buffer[1] = HID_ALLY_FEATURE_CODE_PAGE; + buffer[2] = CMD_SET_XBOX_CONTROLLER; + buffer[3] = 0x01; + buffer[4] = enabled ? 0x01 : 0x00; + + ret = ally_gamepad_send_one_byte_packet( + ally, hdev, CMD_SET_XBOX_CONTROLLER, + enabled ? 0x01 : 0x00); + if (ret < 0) return ret; + + cfg->xbox_controller_enabled = enabled; + return 0; +} + +static ssize_t xbox_controller_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg; + + if (!ally || !ally->config) + return -ENODEV; + + cfg = ally->config; + + if (!cfg->xbox_controller_support) + return sprintf(buf, "Unsupported\n"); + + return sprintf(buf, "%d\n", cfg->xbox_controller_enabled); +} + +static ssize_t xbox_controller_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg; + bool enabled; + int ret; + + cfg = ally->config; + if (!cfg->xbox_controller_support) + return -ENODEV; + + ret = kstrtobool(buf, &enabled); + if (ret) + return ret; + + ret = ally_set_xbox_controller(hdev, cfg, enabled); + + if (ret < 0) + return ret; + + return count; +} + +static DEVICE_ATTR_RW(xbox_controller); + +/** + * ally_set_vibration_intensity - Set vibration intensity values + * @hdev: HID device + * @cfg: Ally config + * @left: Left motor intensity (0-100) + * @right: Right motor intensity (0-100) + * + * Returns 0 on success, negative error code on failure + */ +static int ally_set_vibration_intensity(struct hid_device *hdev, + struct ally_config *cfg, u8 left, + u8 right) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + u8 buffer[64] = { 0 }; + int ret; + + if (!cfg) + return -ENODEV; + + buffer[0] = HID_ALLY_SET_REPORT_ID; + buffer[1] = HID_ALLY_FEATURE_CODE_PAGE; + buffer[2] = CMD_SET_VIBRATION_INTENSITY; + buffer[3] = 0x02; /* Length */ + buffer[4] = left; + buffer[5] = right; + + ret = ally_gamepad_send_two_byte_packet( + ally, hdev, CMD_SET_VIBRATION_INTENSITY, left, right); + if (ret < 0) + return ret; + + mutex_lock(&cfg->config_mutex); + cfg->vibration_intensity_left = left; + cfg->vibration_intensity_right = right; + mutex_unlock(&cfg->config_mutex); + + return 0; +} + +static ssize_t vibration_intensity_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg; + + if (!ally || !ally->config) + return -ENODEV; + + cfg = ally->config; + + return sprintf(buf, "%u,%u\n", cfg->vibration_intensity_left, + cfg->vibration_intensity_right); +} + +static ssize_t vibration_intensity_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg; + u8 left, right; + int ret; + + if (!ally || !ally->config) + return -ENODEV; + + cfg = ally->config; + + ret = sscanf(buf, "%hhu %hhu", &left, &right); + if (ret != 2 || left > 100 || right > 100) + return -EINVAL; + + ret = ally_set_vibration_intensity(hdev, cfg, left, right); + if (ret < 0) + return ret; + + return count; +} + +static DEVICE_ATTR_RW(vibration_intensity); + +/** + * ally_set_dzot_ranges - Generic function to set joystick or trigger ranges + * @hdev: HID device + * @cfg: Ally config struct + * @command: Command to use (CMD_SET_JOYSTICK_DEADZONE or CMD_SET_TRIGGER_RANGE) + * @param1: First parameter + * @param2: Second parameter + * @param3: Third parameter + * @param4: Fourth parameter + * + * Returns 0 on success, negative error code on failure + */ +static int ally_set_dzot_ranges(struct hid_device *hdev, + struct ally_config *cfg, + u8 command, u8 param1, u8 param2, + u8 param3, u8 param4) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + u8 packet[HID_ALLY_REPORT_SIZE] = { 0 }; + int ret; + + packet[0] = HID_ALLY_SET_REPORT_ID; + packet[1] = HID_ALLY_FEATURE_CODE_PAGE; + packet[2] = command; + packet[3] = 0x04; /* Length */ + packet[4] = param1; + packet[5] = param2; + packet[6] = param3; + packet[7] = param4; + + ret = ally_gamepad_send_packet(ally, hdev, packet, + HID_ALLY_REPORT_SIZE); + return ret; +} + +static int ally_validate_joystick_dzot(u8 left_dz, u8 left_ot, u8 right_dz, + u8 right_ot) +{ + if (left_dz > 50 || right_dz > 50) + return -EINVAL; + + if (left_ot < 70 || left_ot > 100 || right_ot < 70 || right_ot > 100) + return -EINVAL; + + return 0; +} + +static int ally_set_joystick_dzot(struct hid_device *hdev, + struct ally_config *cfg, u8 left_dz, + u8 left_ot, u8 right_dz, u8 right_ot) +{ + int ret; + + ret = ally_validate_joystick_dzot(left_dz, left_ot, right_dz, right_ot); + if (ret < 0) + return ret; + + ret = ally_set_dzot_ranges(hdev, cfg, + CMD_SET_JOYSTICK_DEADZONE, + left_dz, left_ot, right_dz, + right_ot); + if (ret < 0) + return ret; + + mutex_lock(&cfg->config_mutex); + cfg->left_deadzone = left_dz; + cfg->left_outer_threshold = left_ot; + cfg->right_deadzone = right_dz; + cfg->right_outer_threshold = right_ot; + mutex_unlock(&cfg->config_mutex); + + return 0; +} + +static ssize_t joystick_deadzone_show(struct device *dev, + struct device_attribute *attr, char *buf, + u8 deadzone, u8 outer_threshold) +{ + return sprintf(buf, "%u %u\n", deadzone, outer_threshold); +} + +static ssize_t joystick_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count, + bool is_left, struct ally_config *cfg) +{ + struct hid_device *hdev = to_hid_device(dev); + u8 dz, ot; + int ret; + + ret = sscanf(buf, "%hhu %hhu", &dz, &ot); + if (ret != 2) + return -EINVAL; + + if (is_left) { + ret = ally_set_joystick_dzot(hdev, cfg, dz, ot, + cfg->right_deadzone, + cfg->right_outer_threshold); + } else { + ret = ally_set_joystick_dzot(hdev, cfg, cfg->left_deadzone, + cfg->left_outer_threshold, dz, ot); + } + + if (ret < 0) + return ret; + + return count; +} + +static ssize_t joystick_left_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg = ally->config; + + return joystick_deadzone_show(dev, attr, buf, cfg->left_deadzone, + cfg->left_outer_threshold); +} + +static ssize_t joystick_left_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + return joystick_deadzone_store(dev, attr, buf, count, true, + ally->config); +} + +static ssize_t joystick_right_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg = ally->config; + + return joystick_deadzone_show(dev, attr, buf, cfg->right_deadzone, + cfg->right_outer_threshold); +} + +static ssize_t joystick_right_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + return joystick_deadzone_store(dev, attr, buf, count, false, + ally->config); +} + +ALLY_DEVICE_CONST_ATTR_RO(js_deadzone_index, deadzone_index, "inner outer\n"); +ALLY_DEVICE_CONST_ATTR_RO(js_deadzone_inner_min, deadzone_inner_min, "0\n"); +ALLY_DEVICE_CONST_ATTR_RO(js_deadzone_inner_max, deadzone_inner_max, "50\n"); +ALLY_DEVICE_CONST_ATTR_RO(js_deadzone_outer_min, deadzone_outer_min, "70\n"); +ALLY_DEVICE_CONST_ATTR_RO(js_deadzone_outer_max, deadzone_outer_max, "100\n"); + +ALLY_DEVICE_ATTR_RW(joystick_left_deadzone, deadzone); +ALLY_DEVICE_ATTR_RW(joystick_right_deadzone, deadzone); + +/** + * ally_set_anti_deadzone - Set anti-deadzone values for joysticks + * @ally: ally handheld structure + * @left_adz: Left joystick anti-deadzone value (0-100) + * @right_adz: Right joystick anti-deadzone value (0-100) + * + * Return: 0 on success, negative on failure + */ +static int ally_set_anti_deadzone(struct ally_handheld *ally, u8 left_adz, + u8 right_adz) +{ + struct hid_device *hdev = ally->cfg_hdev; + int ret; + + if (!ally->config->anti_deadzone_support) { + hid_dbg(hdev, "Anti-deadzone not supported on this device\n"); + return -EOPNOTSUPP; + } + + if (left_adz > 100 || right_adz > 100) + return -EINVAL; + + ret = ally_gamepad_send_two_byte_packet( + ally, hdev, CMD_SET_ANTI_DEADZONE, left_adz, right_adz); + if (ret < 0) { + hid_err(hdev, "Failed to set anti-deadzone values: %d\n", ret); + return ret; + } + + ally->config->left_anti_deadzone = left_adz; + ally->config->right_anti_deadzone = right_adz; + hid_dbg(hdev, "Set joystick anti-deadzone: left=%d, right=%d\n", + left_adz, right_adz); + + return 0; +} + +static ssize_t anti_deadzone_show(struct device *dev, + struct device_attribute *attr, char *buf, + u8 anti_deadzone) +{ + return sprintf(buf, "%u\n", anti_deadzone); +} + +static ssize_t anti_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count, bool is_left, + struct ally_handheld *ally) +{ + u8 adz; + int ret; + + if (!ally || !ally->config) + return -ENODEV; + + if (!ally->config->anti_deadzone_support) + return -EOPNOTSUPP; + + ret = kstrtou8(buf, 10, &adz); + if (ret) + return ret; + + if (adz > 100) + return -EINVAL; + + if (is_left) + ret = ally_set_anti_deadzone(ally, adz, ally->config->right_anti_deadzone); + else + ret = ally_set_anti_deadzone(ally, ally->config->left_anti_deadzone, adz); + + if (ret < 0) + return ret; + + return count; +} + +static ssize_t js_left_anti_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + if (!ally || !ally->config) + return -ENODEV; + + return anti_deadzone_show(dev, attr, buf, + ally->config->left_anti_deadzone); +} + +static ssize_t js_left_anti_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + return anti_deadzone_store(dev, attr, buf, count, true, ally); +} + +static ssize_t js_right_anti_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + if (!ally || !ally->config) + return -ENODEV; + + return anti_deadzone_show(dev, attr, buf, + ally->config->right_anti_deadzone); +} + +static ssize_t js_right_anti_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + return anti_deadzone_store(dev, attr, buf, count, false, ally); +} + +ALLY_DEVICE_ATTR_RW(js_left_anti_deadzone, anti_deadzone); +ALLY_DEVICE_ATTR_RW(js_right_anti_deadzone, anti_deadzone); +ALLY_DEVICE_CONST_ATTR_RO(js_anti_deadzone_min, js_anti_deadzone_min, "0\n"); +ALLY_DEVICE_CONST_ATTR_RO(js_anti_deadzone_max, js_anti_deadzone_max, "100\n"); + +/** + * ally_set_joystick_resp_curve - Set joystick response curve parameters + * @ally: ally handheld structure + * @hdev: HID device + * @side: Which joystick side (0=left, 1=right) + * @curve: Response curve parameter structure + * + * Return: 0 on success, negative on failure + */ +static int ally_set_joystick_resp_curve(struct ally_handheld *ally, + struct hid_device *hdev, u8 side, + struct joystick_resp_curve *curve) +{ + u8 packet[HID_ALLY_REPORT_SIZE] = { 0 }; + int ret; + struct ally_config *cfg = ally->config; + + if (!cfg || !cfg->resp_curve_support) { + hid_dbg(hdev, "Response curve not supported on this device\n"); + return -EOPNOTSUPP; + } + + if (side > 1) { + return -EINVAL; + } + + packet[0] = HID_ALLY_SET_REPORT_ID; + packet[1] = HID_ALLY_FEATURE_CODE_PAGE; + packet[2] = CMD_SET_RESP_CURVE; + packet[3] = 0x09; /* Length */ + packet[4] = side; + + packet[5] = curve->entry_1.move; + packet[6] = curve->entry_1.resp; + packet[7] = curve->entry_2.move; + packet[8] = curve->entry_2.resp; + packet[9] = curve->entry_3.move; + packet[10] = curve->entry_3.resp; + packet[11] = curve->entry_4.move; + packet[12] = curve->entry_4.resp; + + ret = ally_gamepad_send_packet(ally, hdev, packet, + HID_ALLY_REPORT_SIZE); + if (ret < 0) { + hid_err(hdev, "Failed to set joystick response curve: %d\n", + ret); + return ret; + } + + mutex_lock(&cfg->config_mutex); + if (side == 0) { + memcpy(&cfg->left_curve, curve, sizeof(*curve)); + } else { + memcpy(&cfg->right_curve, curve, sizeof(*curve)); + } + mutex_unlock(&cfg->config_mutex); + + hid_dbg(hdev, "Set joystick response curve for side %d\n", side); + return 0; +} + +static int response_curve_apply(struct hid_device *hdev, struct ally_handheld *ally, bool is_left) +{ + struct ally_config *cfg = ally->config; + struct joystick_resp_curve *curve = is_left ? &cfg->left_curve : &cfg->right_curve; + + if (!(curve->entry_1.move < curve->entry_2.move && + curve->entry_2.move < curve->entry_3.move && + curve->entry_3.move < curve->entry_4.move)) { + return -EINVAL; + } + + return ally_set_joystick_resp_curve(ally, hdev, is_left ? 0 : 1, curve); +} + +static ssize_t response_curve_apply_left_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + int ret; + bool apply; + + if (!ally->config->resp_curve_support) + return -EOPNOTSUPP; + + ret = kstrtobool(buf, &apply); + if (ret) + return ret; + + if (!apply) + return count; /* Only apply on "1" or "true" value */ + + ret = response_curve_apply(hdev, ally, true); + if (ret < 0) + return ret; + + return count; +} + +static ssize_t response_curve_apply_right_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + int ret; + bool apply; + + if (!ally->config->resp_curve_support) + return -EOPNOTSUPP; + + ret = kstrtobool(buf, &apply); + if (ret) + return ret; + + if (!apply) + return count; /* Only apply on "1" or "true" value */ + + ret = response_curve_apply(hdev, ally, false); + if (ret < 0) + return ret; + + return count; +} + +static ssize_t response_curve_pct_show(struct device *dev, + struct device_attribute *attr, char *buf, + struct joystick_resp_curve *curve, int idx) +{ + switch (idx) { + case 1: return sprintf(buf, "%u\n", curve->entry_1.resp); + case 2: return sprintf(buf, "%u\n", curve->entry_2.resp); + case 3: return sprintf(buf, "%u\n", curve->entry_3.resp); + case 4: return sprintf(buf, "%u\n", curve->entry_4.resp); + default: return -EINVAL; + } +} + +static ssize_t response_curve_move_show(struct device *dev, + struct device_attribute *attr, char *buf, + struct joystick_resp_curve *curve, int idx) +{ + switch (idx) { + case 1: return sprintf(buf, "%u\n", curve->entry_1.move); + case 2: return sprintf(buf, "%u\n", curve->entry_2.move); + case 3: return sprintf(buf, "%u\n", curve->entry_3.move); + case 4: return sprintf(buf, "%u\n", curve->entry_4.move); + default: return -EINVAL; + } +} + +static ssize_t response_curve_pct_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count, + bool is_left, struct ally_handheld *ally, + int idx) +{ + struct ally_config *cfg = ally->config; + struct joystick_resp_curve *curve = is_left ? &cfg->left_curve : &cfg->right_curve; + u8 value; + int ret; + + if (!cfg->resp_curve_support) + return -EOPNOTSUPP; + + ret = kstrtou8(buf, 10, &value); + if (ret) + return ret; + + if (value > 100) + return -EINVAL; + + mutex_lock(&cfg->config_mutex); + switch (idx) { + case 1: curve->entry_1.resp = value; break; + case 2: curve->entry_2.resp = value; break; + case 3: curve->entry_3.resp = value; break; + case 4: curve->entry_4.resp = value; break; + default: ret = -EINVAL; + } + mutex_unlock(&cfg->config_mutex); + + if (ret < 0) + return ret; + + return count; +} + +static ssize_t response_curve_move_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count, + bool is_left, struct ally_handheld *ally, + int idx) +{ + struct ally_config *cfg = ally->config; + struct joystick_resp_curve *curve = is_left ? &cfg->left_curve : &cfg->right_curve; + u8 value; + int ret; + + if (!cfg->resp_curve_support) + return -EOPNOTSUPP; + + ret = kstrtou8(buf, 10, &value); + if (ret) + return ret; + + if (value > 100) + return -EINVAL; + + mutex_lock(&cfg->config_mutex); + switch (idx) { + case 1: curve->entry_1.move = value; break; + case 2: curve->entry_2.move = value; break; + case 3: curve->entry_3.move = value; break; + case 4: curve->entry_4.move = value; break; + default: ret = -EINVAL; + } + mutex_unlock(&cfg->config_mutex); + + if (ret < 0) + return ret; + + return count; +} + +#define DEFINE_JS_CURVE_PCT_FOPS(region, side) \ + static ssize_t response_curve_pct_##region##_##side##_show( \ + struct device *dev, struct device_attribute *attr, char *buf) \ + { \ + struct hid_device *hdev = to_hid_device(dev); \ + struct ally_handheld *ally = hid_get_drvdata(hdev); \ + return response_curve_pct_show( \ + dev, attr, buf, &ally->config->side##_curve, region); \ + } \ + \ + static ssize_t response_curve_pct_##region##_##side##_store( \ + struct device *dev, struct device_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct hid_device *hdev = to_hid_device(dev); \ + struct ally_handheld *ally = hid_get_drvdata(hdev); \ + return response_curve_pct_store(dev, attr, buf, count, \ + side##_is_left, ally, region); \ + } + +#define DEFINE_JS_CURVE_MOVE_FOPS(region, side) \ + static ssize_t response_curve_move_##region##_##side##_show( \ + struct device *dev, struct device_attribute *attr, char *buf) \ + { \ + struct hid_device *hdev = to_hid_device(dev); \ + struct ally_handheld *ally = hid_get_drvdata(hdev); \ + return response_curve_move_show( \ + dev, attr, buf, &ally->config->side##_curve, region); \ + } \ + \ + static ssize_t response_curve_move_##region##_##side##_store( \ + struct device *dev, struct device_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct hid_device *hdev = to_hid_device(dev); \ + struct ally_handheld *ally = hid_get_drvdata(hdev); \ + return response_curve_move_store(dev, attr, buf, count, \ + side##_is_left, ally, region); \ + } + +#define DEFINE_JS_CURVE_ATTRS(region, side) \ + DEFINE_JS_CURVE_PCT_FOPS(region, side) \ + DEFINE_JS_CURVE_MOVE_FOPS(region, side) \ + ALLY_DEVICE_ATTR_RW(response_curve_pct_##region##_##side, \ + response_curve_pct_##region); \ + ALLY_DEVICE_ATTR_RW(response_curve_move_##region##_##side, \ + response_curve_move_##region) + +/* Helper defines for "is_left" parameter */ +#define left_is_left true +#define right_is_left false + +DEFINE_JS_CURVE_ATTRS(1, left); +DEFINE_JS_CURVE_ATTRS(2, left); +DEFINE_JS_CURVE_ATTRS(3, left); +DEFINE_JS_CURVE_ATTRS(4, left); + +DEFINE_JS_CURVE_ATTRS(1, right); +DEFINE_JS_CURVE_ATTRS(2, right); +DEFINE_JS_CURVE_ATTRS(3, right); +DEFINE_JS_CURVE_ATTRS(4, right); + +ALLY_DEVICE_ATTR_WO(response_curve_apply_left, response_curve_apply); +ALLY_DEVICE_ATTR_WO(response_curve_apply_right, response_curve_apply); + +static ssize_t deadzone_left_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg = ally->config; + + return sprintf(buf, "%u %u\n", cfg->left_deadzone, cfg->left_outer_threshold); +} + +static ssize_t deadzone_right_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg = ally->config; + + return sprintf(buf, "%u %u\n", cfg->right_deadzone, cfg->right_outer_threshold); +} + +DEVICE_ATTR_RO(deadzone_left); +DEVICE_ATTR_RO(deadzone_right); +ALLY_DEVICE_CONST_ATTR_RO(deadzone_index, deadzone_index, "inner outer\n"); + +static struct attribute *axis_xy_left_attrs[] = { + &dev_attr_joystick_left_deadzone.attr, + &dev_attr_js_deadzone_index.attr, + &dev_attr_js_deadzone_inner_min.attr, + &dev_attr_js_deadzone_inner_max.attr, + &dev_attr_js_deadzone_outer_min.attr, + &dev_attr_js_deadzone_outer_max.attr, + &dev_attr_js_left_anti_deadzone.attr, + &dev_attr_js_anti_deadzone_min.attr, + &dev_attr_js_anti_deadzone_max.attr, + &dev_attr_response_curve_pct_1_left.attr, + &dev_attr_response_curve_pct_2_left.attr, + &dev_attr_response_curve_pct_3_left.attr, + &dev_attr_response_curve_pct_4_left.attr, + &dev_attr_response_curve_move_1_left.attr, + &dev_attr_response_curve_move_2_left.attr, + &dev_attr_response_curve_move_3_left.attr, + &dev_attr_response_curve_move_4_left.attr, + &dev_attr_response_curve_apply_left.attr, + NULL +}; + +static struct attribute *axis_xy_right_attrs[] = { + &dev_attr_joystick_right_deadzone.attr, + &dev_attr_js_deadzone_index.attr, + &dev_attr_js_deadzone_inner_min.attr, + &dev_attr_js_deadzone_inner_max.attr, + &dev_attr_js_deadzone_outer_min.attr, + &dev_attr_js_deadzone_outer_max.attr, + &dev_attr_js_right_anti_deadzone.attr, + &dev_attr_js_anti_deadzone_min.attr, + &dev_attr_js_anti_deadzone_max.attr, + &dev_attr_response_curve_pct_1_right.attr, + &dev_attr_response_curve_pct_2_right.attr, + &dev_attr_response_curve_pct_3_right.attr, + &dev_attr_response_curve_pct_4_right.attr, + &dev_attr_response_curve_move_1_right.attr, + &dev_attr_response_curve_move_2_right.attr, + &dev_attr_response_curve_move_3_right.attr, + &dev_attr_response_curve_move_4_right.attr, + &dev_attr_response_curve_apply_right.attr, + NULL +}; + +/** + * ally_set_trigger_range - Set trigger range values + * @hdev: HID device + * @cfg: Ally config + * @left_min: Left trigger minimum (0-255) + * @left_max: Left trigger maximum (0-255) + * @right_min: Right trigger minimum (0-255) + * @right_max: Right trigger maximum (0-255) + * + * Returns 0 on success, negative error code on failure + */ +static int ally_set_trigger_range(struct hid_device *hdev, + struct ally_config *cfg, u8 left_min, + u8 left_max, u8 right_min, u8 right_max) +{ + int ret; + + if (left_min >= left_max || right_min >= right_max) + return -EINVAL; + + ret = ally_set_dzot_ranges(hdev, cfg, + CMD_SET_TRIGGER_RANGE, + left_min, left_max, right_min, + right_max); + if (ret < 0) + return ret; + + mutex_lock(&cfg->config_mutex); + cfg->left_trigger_min = left_min; + cfg->left_trigger_max = left_max; + cfg->right_trigger_min = right_min; + cfg->right_trigger_max = right_max; + mutex_unlock(&cfg->config_mutex); + + return 0; +} + +static ssize_t trigger_range_show(struct device *dev, + struct device_attribute *attr, char *buf, + u8 min_val, u8 max_val) +{ + return sprintf(buf, "%u %u\n", min_val, max_val); +} + +static ssize_t trigger_range_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count, bool is_left, + struct ally_config *cfg) +{ + struct hid_device *hdev = to_hid_device(dev); + u8 min_val, max_val; + int ret; + + ret = sscanf(buf, "%hhu %hhu", &min_val, &max_val); + if (ret != 2) + return -EINVAL; + + if (is_left) { + ret = ally_set_trigger_range(hdev, cfg, min_val, max_val, + cfg->right_trigger_min, + cfg->right_trigger_max); + } else { + ret = ally_set_trigger_range(hdev, cfg, cfg->left_trigger_min, + cfg->left_trigger_max, min_val, + max_val); + } + + if (ret < 0) + return ret; + + return count; +} + +static ssize_t trigger_left_deadzone_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg = ally->config; + + return trigger_range_show(dev, attr, buf, cfg->left_trigger_min, + cfg->left_trigger_max); +} + +static ssize_t trigger_left_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + return trigger_range_store(dev, attr, buf, count, true, ally->config); +} + +static ssize_t trigger_right_deadzone_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg = ally->config; + + return trigger_range_show(dev, attr, buf, cfg->right_trigger_min, + cfg->right_trigger_max); +} + +static ssize_t trigger_right_deadzone_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + return trigger_range_store(dev, attr, buf, count, false, ally->config); +} + +ALLY_DEVICE_CONST_ATTR_RO(tr_deadzone_inner_min, deadzone_inner_min, "0\n"); +ALLY_DEVICE_CONST_ATTR_RO(tr_deadzone_inner_max, deadzone_inner_max, "255\n"); + +ALLY_DEVICE_ATTR_RW(trigger_left_deadzone, deadzone); +ALLY_DEVICE_ATTR_RW(trigger_right_deadzone, deadzone); + +static struct attribute *axis_z_left_attrs[] = { + &dev_attr_trigger_left_deadzone.attr, + &dev_attr_tr_deadzone_inner_min.attr, + &dev_attr_tr_deadzone_inner_max.attr, + NULL +}; + +static struct attribute *axis_z_right_attrs[] = { + &dev_attr_trigger_right_deadzone.attr, + &dev_attr_tr_deadzone_inner_min.attr, + &dev_attr_tr_deadzone_inner_max.attr, + NULL +}; + +/* Map from string name to enum value */ +static int get_gamepad_mode_from_name(const char *name) +{ + int i; + + for (i = ALLY_GAMEPAD_MODE_GAMEPAD; i <= ALLY_GAMEPAD_MODE_KEYBOARD; + i++) { + if (gamepad_mode_names[i] && + strcmp(name, gamepad_mode_names[i]) == 0) + return i; + } + + return -1; +} + +/** + * ally_set_gamepad_mode - Set the gamepad operating mode + * @ally: ally handheld structure + * @hdev: HID device + * @mode: Gamepad mode to set + * + * Returns: 0 on success, negative on failure + */ +static int ally_set_gamepad_mode(struct ally_handheld *ally, + struct hid_device *hdev, u8 mode) +{ + struct ally_config *cfg = ally->config; + int ret; + + if (!cfg) + return -EINVAL; + + if (mode < ALLY_GAMEPAD_MODE_GAMEPAD || + mode > ALLY_GAMEPAD_MODE_KEYBOARD) { + hid_err(hdev, "Invalid gamepad mode: %u\n", mode); + return -EINVAL; + } + + ret = ally_gamepad_send_one_byte_packet(ally, hdev, + CMD_SET_GAMEPAD_MODE, mode); + if (ret < 0) { + hid_err(hdev, "Failed to set gamepad mode: %d\n", ret); + return ret; + } + + mutex_lock(&cfg->config_mutex); + cfg->gamepad_mode = mode; + mutex_unlock(&cfg->config_mutex); + + hid_info(hdev, "Set gamepad mode to %s\n", gamepad_mode_names[mode]); + return 0; +} + +static ssize_t gamepad_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_config *cfg; + + if (!ally || !ally->config) + return -ENODEV; + + cfg = ally->config; + + if (cfg->gamepad_mode >= ALLY_GAMEPAD_MODE_GAMEPAD && + cfg->gamepad_mode <= ALLY_GAMEPAD_MODE_KEYBOARD) { + return sprintf(buf, "%s\n", + gamepad_mode_names[cfg->gamepad_mode]); + } else { + return sprintf(buf, "unknown (%u)\n", cfg->gamepad_mode); + } +} + +static ssize_t gamepad_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + char mode_name[16]; + int mode; + int ret; + + if (!ally || !ally->config) + return -ENODEV; + + if (sscanf(buf, "%15s", mode_name) != 1) + return -EINVAL; + + mode = get_gamepad_mode_from_name(mode_name); + if (mode < 0) { + hid_err(hdev, "Unknown gamepad mode: %s\n", mode_name); + return -EINVAL; + } + + ret = ally_set_gamepad_mode(ally, hdev, mode); + if (ret < 0) + return ret; + + return count; +} + +static ssize_t gamepad_modes_available_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i; + int len = 0; + + for (i = ALLY_GAMEPAD_MODE_GAMEPAD; i <= ALLY_GAMEPAD_MODE_KEYBOARD; + i++) { + len += sprintf(buf + len, "%s ", gamepad_mode_names[i]); + } + + /* Replace the last space with a newline */ + if (len > 0) + buf[len - 1] = '\n'; + + return len; +} + +DEVICE_ATTR_RW(gamepad_mode); +DEVICE_ATTR_RO(gamepad_modes_available); + +static int ally_set_default_gamepad_mode(struct hid_device *hdev, + struct ally_config *cfg) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + + cfg->gamepad_mode = ALLY_GAMEPAD_MODE_GAMEPAD; + + return ally_set_gamepad_mode(ally, hdev, cfg->gamepad_mode); +} + +static struct attribute *ally_config_attrs[] = { + &dev_attr_xbox_controller.attr, + &dev_attr_vibration_intensity.attr, + &dev_attr_gamepad_mode.attr, + &dev_attr_gamepad_modes_available.attr, + NULL +}; + +static const struct attribute_group ally_attr_groups[] = { + { + .attrs = ally_config_attrs, + }, + { + .name = "axis_xy_left", + .attrs = axis_xy_left_attrs, + }, + { + .name = "axis_xy_right", + .attrs = axis_xy_right_attrs, + }, + { + .name = "axis_z_left", + .attrs = axis_z_left_attrs, + }, + { + .name = "axis_z_right", + .attrs = axis_z_right_attrs, + }, +}; + +/** + * ally_get_turbo_params - Get turbo parameters for a specific button + * @cfg: Ally config structure + * @button_id: Button identifier from ally_button_id enum + * + * Returns: Pointer to the button's turbo parameters, or NULL if invalid + */ +static struct button_turbo_params *ally_get_turbo_params(struct ally_config *cfg, + enum ally_button_id button_id) +{ + struct turbo_config *turbo; + + if (!cfg || button_id >= ALLY_BTN_MAX) + return NULL; + + turbo = &cfg->turbo; + + switch (button_id) { + case ALLY_BTN_A: + return &turbo->btn_a; + case ALLY_BTN_B: + return &turbo->btn_b; + case ALLY_BTN_X: + return &turbo->btn_x; + case ALLY_BTN_Y: + return &turbo->btn_y; + case ALLY_BTN_LB: + return &turbo->btn_lb; + case ALLY_BTN_RB: + return &turbo->btn_rb; + case ALLY_BTN_DU: + return &turbo->btn_du; + case ALLY_BTN_DD: + return &turbo->btn_dd; + case ALLY_BTN_DL: + return &turbo->btn_dl; + case ALLY_BTN_DR: + return &turbo->btn_dr; + case ALLY_BTN_J0B: + return &turbo->btn_j0b; + case ALLY_BTN_J1B: + return &turbo->btn_j1b; + case ALLY_BTN_MENU: + return &turbo->btn_menu; + case ALLY_BTN_VIEW: + return &turbo->btn_view; + case ALLY_BTN_M1: + return &turbo->btn_m1; + case ALLY_BTN_M2: + return &turbo->btn_m2; + default: + return NULL; + } +} + +/** + * ally_set_turbo_params - Set turbo parameters for all buttons + * @hdev: HID device + * @cfg: Ally config structure + * + * Returns: 0 on success, negative on failure + */ +static int ally_set_turbo_params(struct hid_device *hdev, struct ally_config *cfg) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct turbo_config *turbo = &cfg->turbo; + u8 packet[HID_ALLY_REPORT_SIZE] = { 0 }; + int ret; + + if (!cfg->turbo_support) { + hid_dbg(hdev, "Turbo functionality not supported on this device\n"); + return -EOPNOTSUPP; + } + + packet[0] = HID_ALLY_SET_REPORT_ID; + packet[1] = HID_ALLY_FEATURE_CODE_PAGE; + packet[2] = CMD_SET_TURBO_PARAMS; + packet[3] = 0x20; /* Length - 32 bytes for 16 buttons with 2 values each */ + + packet[4] = turbo->btn_du.turbo; + packet[5] = turbo->btn_du.toggle; + packet[6] = turbo->btn_dd.turbo; + packet[7] = turbo->btn_dd.toggle; + packet[8] = turbo->btn_dl.turbo; + packet[9] = turbo->btn_dl.toggle; + packet[10] = turbo->btn_dr.turbo; + packet[11] = turbo->btn_dr.toggle; + packet[12] = turbo->btn_j0b.turbo; + packet[13] = turbo->btn_j0b.toggle; + packet[14] = turbo->btn_j1b.turbo; + packet[15] = turbo->btn_j1b.toggle; + packet[16] = turbo->btn_lb.turbo; + packet[17] = turbo->btn_lb.toggle; + packet[18] = turbo->btn_rb.turbo; + packet[19] = turbo->btn_rb.toggle; + packet[20] = turbo->btn_a.turbo; + packet[21] = turbo->btn_a.toggle; + packet[22] = turbo->btn_b.turbo; + packet[23] = turbo->btn_b.toggle; + packet[24] = turbo->btn_x.turbo; + packet[25] = turbo->btn_x.toggle; + packet[26] = turbo->btn_y.turbo; + packet[27] = turbo->btn_y.toggle; + packet[28] = turbo->btn_view.turbo; + packet[29] = turbo->btn_view.toggle; + packet[30] = turbo->btn_menu.turbo; + packet[31] = turbo->btn_menu.toggle; + packet[32] = turbo->btn_m2.turbo; + packet[33] = turbo->btn_m2.toggle; + packet[34] = turbo->btn_m1.turbo; + packet[35] = turbo->btn_m1.toggle; + + ret = ally_gamepad_send_packet(ally, hdev, packet, HID_ALLY_REPORT_SIZE); + if (ret < 0) { + hid_err(hdev, "Failed to set turbo parameters: %d\n", ret); + return ret; + } + + return 0; +} + +struct button_turbo_attr { + struct device_attribute dev_attr; + int button_id; +}; + +#define to_button_turbo_attr(x) container_of(x, struct button_turbo_attr, dev_attr) + +static ssize_t button_turbo_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct button_turbo_attr *btn_attr = to_button_turbo_attr(attr); + struct button_turbo_params *params; + + if (!ally->config->turbo_support) + return sprintf(buf, "Unsupported\n"); + + params = ally_get_turbo_params(ally->config, btn_attr->button_id); + if (!params) + return -EINVAL; + + /* Format: turbo_interval_ms[,toggle_interval_ms] */ + if (params->toggle) + return sprintf(buf, "%d,%d\n", params->turbo * 50, params->toggle * 50); + else + return sprintf(buf, "%d\n", params->turbo * 50); +} + +static ssize_t button_turbo_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct button_turbo_attr *btn_attr = to_button_turbo_attr(attr); + struct button_turbo_params *params; + unsigned int turbo_ms, toggle_ms = 0; + int ret; + + if (!ally->config->turbo_support) + return -EOPNOTSUPP; + + params = ally_get_turbo_params(ally->config, btn_attr->button_id); + if (!params) + return -EINVAL; + + /* Parse input: turbo_interval_ms[,toggle_interval_ms] */ + ret = sscanf(buf, "%u,%u", &turbo_ms, &toggle_ms); + if (ret < 1) + return -EINVAL; + + if (turbo_ms != 0 && (turbo_ms < 50 || turbo_ms > 1000)) + return -EINVAL; + + if (ret > 1 && toggle_ms > 0 && (toggle_ms < 50 || toggle_ms > 1000)) + return -EINVAL; + + mutex_lock(&ally->config->config_mutex); + + params->turbo = turbo_ms / 50; + params->toggle = toggle_ms / 50; + + ret = ally_set_turbo_params(hdev, ally->config); + + mutex_unlock(&ally->config->config_mutex); + + if (ret < 0) + return ret; + + return count; +} + +/* Helper to create button turbo attribute */ +static struct button_turbo_attr *button_turbo_attr_create(int button_id) +{ + struct button_turbo_attr *attr; + + attr = kzalloc(sizeof(*attr), GFP_KERNEL); + if (!attr) + return NULL; + + attr->button_id = button_id; + sysfs_attr_init(&attr->dev_attr.attr); + attr->dev_attr.attr.name = "turbo"; + attr->dev_attr.attr.mode = 0644; + attr->dev_attr.show = button_turbo_show; + attr->dev_attr.store = button_turbo_store; + + return attr; +} + +/* Button remap attribute structure */ +struct button_remap_attr { + struct device_attribute dev_attr; + enum ally_button_id button_id; + bool is_macro; +}; + +#define to_button_remap_attr(x) container_of(x, struct button_remap_attr, dev_attr) + +/* Get appropriate button pair index and position for a given button */ +static int get_button_pair_info(enum ally_button_id button_id, + enum btn_pair_index *pair_idx, + bool *is_first) +{ + switch (button_id) { + case ALLY_BTN_DU: + *pair_idx = BTN_PAIR_DPAD_UPDOWN; + *is_first = true; + break; + case ALLY_BTN_DD: + *pair_idx = BTN_PAIR_DPAD_UPDOWN; + *is_first = false; + break; + case ALLY_BTN_DL: + *pair_idx = BTN_PAIR_DPAD_LEFTRIGHT; + *is_first = true; + break; + case ALLY_BTN_DR: + *pair_idx = BTN_PAIR_DPAD_LEFTRIGHT; + *is_first = false; + break; + case ALLY_BTN_J0B: + *pair_idx = BTN_PAIR_STICK_LR; + *is_first = true; + break; + case ALLY_BTN_J1B: + *pair_idx = BTN_PAIR_STICK_LR; + *is_first = false; + break; + case ALLY_BTN_LB: + *pair_idx = BTN_PAIR_BUMPER_LR; + *is_first = true; + break; + case ALLY_BTN_RB: + *pair_idx = BTN_PAIR_BUMPER_LR; + *is_first = false; + break; + case ALLY_BTN_A: + *pair_idx = BTN_PAIR_AB; + *is_first = true; + break; + case ALLY_BTN_B: + *pair_idx = BTN_PAIR_AB; + *is_first = false; + break; + case ALLY_BTN_X: + *pair_idx = BTN_PAIR_XY; + *is_first = true; + break; + case ALLY_BTN_Y: + *pair_idx = BTN_PAIR_XY; + *is_first = false; + break; + case ALLY_BTN_VIEW: + *pair_idx = BTN_PAIR_VIEW_MENU; + *is_first = true; + break; + case ALLY_BTN_MENU: + *pair_idx = BTN_PAIR_VIEW_MENU; + *is_first = false; + break; + case ALLY_BTN_M1: + *pair_idx = BTN_PAIR_M1M2; + *is_first = true; + break; + case ALLY_BTN_M2: + *pair_idx = BTN_PAIR_M1M2; + *is_first = false; + break; + default: + return -EINVAL; + } + + return 0; +} + +static ssize_t button_remap_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct button_remap_attr *btn_attr = to_button_remap_attr(attr); + struct ally_config *cfg = ally->config; + enum ally_button_id button_id = btn_attr->button_id; + enum btn_pair_index pair_idx; + bool is_first; + struct button_pair_map *pair; + struct button_map *btn_map; + int ret; + + if (!cfg) + return -ENODEV; + + ret = get_button_pair_info(button_id, &pair_idx, &is_first); + if (ret < 0) + return ret; + + mutex_lock(&cfg->config_mutex); + pair = &((struct ally_button_mapping + *)(cfg->button_mappings))[cfg->gamepad_mode] + .button_pairs[pair_idx - 1]; + btn_map = is_first ? &pair->first : &pair->second; + + if (btn_attr->is_macro) { + if (btn_map->macro->type == BTN_TYPE_NONE) + ret = sprintf(buf, "NONE\n"); + else + ret = sprintf(buf, "%s\n", btn_map->macro->name); + } else { + if (btn_map->remap->type == BTN_TYPE_NONE) + ret = sprintf(buf, "NONE\n"); + else + ret = sprintf(buf, "%s\n", btn_map->remap->name); + } + mutex_unlock(&cfg->config_mutex); + + return ret; +} + +static ssize_t button_remap_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct button_remap_attr *btn_attr = to_button_remap_attr(attr); + struct ally_config *cfg = ally->config; + enum ally_button_id button_id = btn_attr->button_id; + enum btn_pair_index pair_idx; + bool is_first; + struct button_pair_map *pair; + struct button_map *btn_map; + char btn_name[32]; + const struct btn_code_map *code; + int ret; + + if (!cfg) + return -ENODEV; + + if (sscanf(buf, "%31s", btn_name) != 1) + return -EINVAL; + + /* Handle "NONE" specially */ + if (strcmp(btn_name, "NONE") == 0) { + code = &ally_btn_codes[0]; /* NONE entry */ + } else { + code = find_button_by_name(btn_name); + if (!code) + return -EINVAL; + } + + ret = get_button_pair_info(button_id, &pair_idx, &is_first); + if (ret < 0) + return ret; + + mutex_lock(&cfg->config_mutex); + /* Access the mapping for current gamepad mode */ + pair = &((struct ally_button_mapping + *)(cfg->button_mappings))[cfg->gamepad_mode] + .button_pairs[pair_idx - 1]; + btn_map = is_first ? &pair->first : &pair->second; + + if (btn_attr->is_macro) { + btn_map->macro = (struct btn_code_map *)code; + } else { + btn_map->remap = (struct btn_code_map *)code; + } + + /* Update pair index */ + pair->pair_index = pair_idx; + + /* Send mapping to device */ + ret = ally_set_button_mapping(hdev, ally, pair); + mutex_unlock(&cfg->config_mutex); + + if (ret < 0) + return ret; + + return count; +} + +/* Helper to create button remap attribute */ +static struct button_remap_attr *button_remap_attr_create(enum ally_button_id button_id, bool is_macro) +{ + struct button_remap_attr *attr; + + attr = kzalloc(sizeof(*attr), GFP_KERNEL); + if (!attr) + return NULL; + + attr->button_id = button_id; + attr->is_macro = is_macro; + sysfs_attr_init(&attr->dev_attr.attr); + attr->dev_attr.attr.name = is_macro ? "macro" : "remap"; + attr->dev_attr.attr.mode = 0644; + attr->dev_attr.show = button_remap_show; + attr->dev_attr.store = button_remap_store; + + return attr; +} + +/* Structure to hold button sysfs information */ +struct button_sysfs_entry { + struct attribute_group group; + struct attribute *attrs[4]; /* turbo + remap + macro + NULL terminator */ + struct button_turbo_attr *turbo_attr; + struct button_remap_attr *remap_attr; + struct button_remap_attr *macro_attr; +}; + +static void ally_set_default_gamepad_mapping(struct ally_button_mapping *mappings) +{ + struct ally_button_mapping *map = &mappings[ALLY_GAMEPAD_MODE_GAMEPAD]; + int i; + + /* Set all pair indexes and initialize to NONE */ + for (i = 0; i < 9; i++) { + map->button_pairs[i].pair_index = i + 1; + map->button_pairs[i].first.remap = + (struct btn_code_map *)&ally_btn_codes[0]; + map->button_pairs[i].first.macro = + (struct btn_code_map *)&ally_btn_codes[0]; + map->button_pairs[i].second.remap = + (struct btn_code_map *)&ally_btn_codes[0]; + map->button_pairs[i].second.macro = + (struct btn_code_map *)&ally_btn_codes[0]; + } + + /* Set direct mappings using array indices */ + map->button_pairs[BTN_PAIR_AB - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[1]; /* PAD_A */ + map->button_pairs[BTN_PAIR_AB - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[2]; /* PAD_B */ + + map->button_pairs[BTN_PAIR_XY - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[3]; /* PAD_X */ + map->button_pairs[BTN_PAIR_XY - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[4]; /* PAD_Y */ + + map->button_pairs[BTN_PAIR_BUMPER_LR - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[5]; /* PAD_LB */ + map->button_pairs[BTN_PAIR_BUMPER_LR - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[6]; /* PAD_RB */ + + map->button_pairs[BTN_PAIR_STICK_LR - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[7]; /* PAD_LS */ + map->button_pairs[BTN_PAIR_STICK_LR - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[8]; /* PAD_RS */ + + map->button_pairs[BTN_PAIR_DPAD_UPDOWN - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[9]; /* PAD_DPAD_UP */ + map->button_pairs[BTN_PAIR_DPAD_UPDOWN - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[10]; /* PAD_DPAD_DOWN */ + + map->button_pairs[BTN_PAIR_DPAD_LEFTRIGHT - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[11]; /* PAD_DPAD_LEFT */ + map->button_pairs[BTN_PAIR_DPAD_LEFTRIGHT - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[12]; /* PAD_DPAD_RIGHT */ + + map->button_pairs[BTN_PAIR_TRIGGER_LR - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[13]; /* PAD_LT */ + map->button_pairs[BTN_PAIR_TRIGGER_LR - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[14]; /* PAD_RT */ + + map->button_pairs[BTN_PAIR_VIEW_MENU - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[15]; /* PAD_VIEW */ + map->button_pairs[BTN_PAIR_VIEW_MENU - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[16]; /* PAD_MENU */ + + map->button_pairs[BTN_PAIR_M1M2 - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[19]; /* KB_M1 */ + map->button_pairs[BTN_PAIR_M1M2 - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[18]; /* KB_M2 */ +} + +static void ally_set_default_keyboard_mapping(struct ally_button_mapping *mappings) +{ + struct ally_button_mapping *map = &mappings[ALLY_GAMEPAD_MODE_KEYBOARD]; + int i; + + /* Set all pair indexes and initialize to NONE */ + for (i = 0; i < 9; i++) { + map->button_pairs[i].pair_index = i + 1; + map->button_pairs[i].first.remap = + (struct btn_code_map *)&ally_btn_codes[0]; + map->button_pairs[i].first.macro = + (struct btn_code_map *)&ally_btn_codes[0]; + map->button_pairs[i].second.remap = + (struct btn_code_map *)&ally_btn_codes[0]; + map->button_pairs[i].second.macro = + (struct btn_code_map *)&ally_btn_codes[0]; + } + + /* Set direct mappings using array indices */ + map->button_pairs[BTN_PAIR_AB - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[1]; /* PAD_A */ + map->button_pairs[BTN_PAIR_AB - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[2]; /* PAD_B */ + + map->button_pairs[BTN_PAIR_XY - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[3]; /* PAD_X */ + map->button_pairs[BTN_PAIR_XY - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[4]; /* PAD_Y */ + + map->button_pairs[BTN_PAIR_BUMPER_LR - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[5]; /* PAD_LB */ + map->button_pairs[BTN_PAIR_BUMPER_LR - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[6]; /* PAD_RB */ + + map->button_pairs[BTN_PAIR_STICK_LR - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[7]; /* PAD_LS */ + map->button_pairs[BTN_PAIR_STICK_LR - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[8]; /* PAD_RS */ + + map->button_pairs[BTN_PAIR_DPAD_UPDOWN - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[9]; /* PAD_DPAD_UP */ + map->button_pairs[BTN_PAIR_DPAD_UPDOWN - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[10]; /* PAD_DPAD_DOWN */ + + map->button_pairs[BTN_PAIR_DPAD_LEFTRIGHT - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[11]; /* PAD_DPAD_LEFT */ + map->button_pairs[BTN_PAIR_DPAD_LEFTRIGHT - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[12]; /* PAD_DPAD_RIGHT */ + + map->button_pairs[BTN_PAIR_TRIGGER_LR - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[13]; /* PAD_LT */ + map->button_pairs[BTN_PAIR_TRIGGER_LR - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[14]; /* PAD_RT */ + + map->button_pairs[BTN_PAIR_VIEW_MENU - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[15]; /* PAD_VIEW */ + map->button_pairs[BTN_PAIR_VIEW_MENU - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[16]; /* PAD_MENU */ + + map->button_pairs[BTN_PAIR_M1M2 - 1].first.remap = + (struct btn_code_map *)&ally_btn_codes[19]; /* KB_M1 */ + map->button_pairs[BTN_PAIR_M1M2 - 1].second.remap = + (struct btn_code_map *)&ally_btn_codes[18]; /* KB_M2 */ +} + +/** + * ally_create_button_attributes - Create button attributes + * @hdev: HID device + * @cfg: Ally config structure + * + * Returns: 0 on success, negative on failure + */ +static int ally_create_button_attributes(struct hid_device *hdev, + struct ally_config *cfg) +{ + struct button_sysfs_entry *entries; + int i, j, ret; + struct ally_button_mapping *mappings; + + entries = devm_kcalloc(&hdev->dev, ALLY_BTN_MAX, sizeof(*entries), + GFP_KERNEL); + if (!entries) + return -ENOMEM; + + /* Allocate mappings for each gamepad mode (1-based indexing) */ + mappings = devm_kcalloc(&hdev->dev, ALLY_GAMEPAD_MODE_KEYBOARD + 1, + sizeof(*mappings), GFP_KERNEL); + if (!mappings) { + ret = -ENOMEM; + goto err_free_entries; + } + + cfg->button_entries = entries; + cfg->button_mappings = mappings; + ally_set_default_gamepad_mapping(mappings); + ally_set_default_keyboard_mapping(mappings); + + for (i = 0; i < ALLY_BTN_MAX; i++) { + if (cfg->turbo_support) { + entries[i].turbo_attr = button_turbo_attr_create(i); + if (!entries[i].turbo_attr) { + ret = -ENOMEM; + goto err_cleanup; + } + } + + entries[i].remap_attr = button_remap_attr_create(i, false); + if (!entries[i].remap_attr) { + ret = -ENOMEM; + goto err_cleanup; + } + + entries[i].macro_attr = button_remap_attr_create(i, true); + if (!entries[i].macro_attr) { + ret = -ENOMEM; + goto err_cleanup; + } + + /* Set up attributes array based on what's supported */ + if (cfg->turbo_support) { + entries[i].attrs[0] = + &entries[i].turbo_attr->dev_attr.attr; + entries[i].attrs[1] = + &entries[i].remap_attr->dev_attr.attr; + entries[i].attrs[2] = + &entries[i].macro_attr->dev_attr.attr; + entries[i].attrs[3] = NULL; + } else { + entries[i].attrs[0] = + &entries[i].remap_attr->dev_attr.attr; + entries[i].attrs[1] = + &entries[i].macro_attr->dev_attr.attr; + entries[i].attrs[2] = NULL; + } + + entries[i].group.name = ally_button_names[i]; + entries[i].group.attrs = entries[i].attrs; + + ret = sysfs_create_group(&hdev->dev.kobj, &entries[i].group); + if (ret < 0) { + hid_err(hdev, + "Failed to create sysfs group for %s: %d\n", + ally_button_names[i], ret); + goto err_cleanup; + } + } + + return 0; + +err_cleanup: + while (--i >= 0) { + sysfs_remove_group(&hdev->dev.kobj, &entries[i].group); + if (entries[i].turbo_attr) + kfree(entries[i].turbo_attr); + if (entries[i].remap_attr) + kfree(entries[i].remap_attr); + if (entries[i].macro_attr) + kfree(entries[i].macro_attr); + } + +err_free_entries: + if (mappings) + devm_kfree(&hdev->dev, mappings); + devm_kfree(&hdev->dev, entries); + return ret; +} + +/** + * ally_remove_button_attributes - Remove button attributes + * @hdev: HID device + * @cfg: Ally config structure + */ +static void ally_remove_button_attributes(struct hid_device *hdev, + struct ally_config *cfg) +{ + struct button_sysfs_entry *entries; + int i; + + if (!cfg || !cfg->button_entries) + return; + + entries = cfg->button_entries; + + /* Remove all attribute groups */ + for (i = 0; i < ALLY_BTN_MAX; i++) { + sysfs_remove_group(&hdev->dev.kobj, &entries[i].group); + if (entries[i].turbo_attr) + kfree(entries[i].turbo_attr); + if (entries[i].remap_attr) + kfree(entries[i].remap_attr); + if (entries[i].macro_attr) + kfree(entries[i].macro_attr); + } + + if (cfg->button_mappings) + devm_kfree(&hdev->dev, cfg->button_mappings); + devm_kfree(&hdev->dev, entries); +} + +/** + * ally_config_create - Initialize configuration and create sysfs entries + * @hdev: HID device + * @ally: Ally device data + * + * Returns 0 on success, negative error code on failure + */ +int ally_config_create(struct hid_device *hdev, struct ally_handheld *ally) +{ + struct ally_config *cfg; + int ret, i; + + if (!hdev || !ally) + return -EINVAL; + + if (get_endpoint_address(hdev) != HID_ALLY_INTF_CFG_IN) + return 0; + + cfg = devm_kzalloc(&hdev->dev, sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return -ENOMEM; + + cfg->hdev = hdev; + + ally->config = cfg; + + ret = ally_detect_capabilities(hdev, cfg); + if (ret < 0) { + hid_err(hdev, "Failed to detect Ally capabilities: %d\n", ret); + goto err_free; + } + + /* Create all attribute groups */ + for (i = 0; i < ARRAY_SIZE(ally_attr_groups); i++) { + ret = sysfs_create_group(&hdev->dev.kobj, &ally_attr_groups[i]); + if (ret < 0) { + hid_err(hdev, "Failed to create sysfs group '%s': %d\n", + ally_attr_groups[i].name, ret); + /* Remove any groups already created */ + while (--i >= 0) + sysfs_remove_group(&hdev->dev.kobj, + &ally_attr_groups[i]); + goto err_free; + } + } + + if (cfg->turbo_support) { + ret = ally_create_button_attributes(hdev, cfg); + if (ret < 0) { + hid_err(hdev, "Failed to create button attributes: %d\n", ret); + for (i = 0; i < ARRAY_SIZE(ally_attr_groups); i++) + sysfs_remove_group(&hdev->dev.kobj, &ally_attr_groups[i]); + goto err_free; + } + } + + ret = ally_set_default_gamepad_mode(hdev, cfg); + if (ret < 0) + hid_warn(hdev, "Failed to set default gamepad mode: %d\n", ret); + + cfg->gamepad_mode = 0x01; + cfg->left_deadzone = 10; + cfg->left_outer_threshold = 90; + cfg->right_deadzone = 10; + cfg->right_outer_threshold = 90; + + cfg->vibration_intensity_left = 100; + cfg->vibration_intensity_right = 100; + cfg->vibration_active = false; + + /* Initialize default response curve values (linear) */ + cfg->left_curve.entry_1.move = 0; + cfg->left_curve.entry_1.resp = 0; + cfg->left_curve.entry_2.move = 33; + cfg->left_curve.entry_2.resp = 33; + cfg->left_curve.entry_3.move = 66; + cfg->left_curve.entry_3.resp = 66; + cfg->left_curve.entry_4.move = 100; + cfg->left_curve.entry_4.resp = 100; + + cfg->right_curve.entry_1.move = 0; + cfg->right_curve.entry_1.resp = 0; + cfg->right_curve.entry_2.move = 33; + cfg->right_curve.entry_2.resp = 33; + cfg->right_curve.entry_3.move = 66; + cfg->right_curve.entry_3.resp = 66; + cfg->right_curve.entry_4.move = 100; + cfg->right_curve.entry_4.resp = 100; + + // ONLY FOR ALLY 1 + if (cfg->xbox_controller_support) { + ret = ally_set_xbox_controller(hdev, cfg, true); + if (ret < 0) + hid_warn( + hdev, + "Failed to set default Xbox controller mode: %d\n", + ret); + } + + cfg->initialized = true; + hid_info(hdev, "Ally configuration system initialized successfully\n"); + + return 0; + +err_free: + ally->config = NULL; + devm_kfree(&hdev->dev, cfg); + return ret; +} + +/** + * ally_config_remove - Clean up configuration resources + * @hdev: HID device + * @ally: Ally device data + */ +void ally_config_remove(struct hid_device *hdev, struct ally_handheld *ally) +{ + struct ally_config *cfg; + int i; + + if (!ally) + return; + + cfg = ally->config; + if (!cfg || !cfg->initialized) + return; + + if (get_endpoint_address(hdev) != HID_ALLY_INTF_CFG_IN) + return; + + if (cfg->turbo_support && cfg->button_entries) + ally_remove_button_attributes(hdev, cfg); + + /* Remove all attribute groups in reverse order */ + for (i = ARRAY_SIZE(ally_attr_groups) - 1; i >= 0; i--) + sysfs_remove_group(&hdev->dev.kobj, &ally_attr_groups[i]); + + ally->config = NULL; + + hid_info(hdev, "Ally configuration system removed\n"); +} diff --git a/drivers/hid/asus-ally-hid/asus-ally-hid-core.c b/drivers/hid/asus-ally-hid/asus-ally-hid-core.c new file mode 100644 index 000000000000..1e8f98b69332 --- /dev/null +++ b/drivers/hid/asus-ally-hid/asus-ally-hid-core.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2023 Luke Jones + */ + +#include "linux/mutex.h" +#include "linux/stddef.h" +#include "linux/types.h" +#include + +#include "../hid-ids.h" +#include "asus-ally.h" + +#define READY_MAX_TRIES 3 + +static const u8 EC_INIT_STRING[] = { 0x5A, 'A', 'S', 'U', 'S', ' ', 'T', 'e','c', 'h', '.', 'I', 'n', 'c', '.', '\0' }; +static const u8 FORCE_FEEDBACK_OFF[] = { 0x0D, 0x0F, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x00, 0xEB }; + +static const struct hid_device_id rog_ally_devices[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X) }, + {} +}; + +const char * ally_keyboard_name = "ROG Ally Keyboard"; +const char * ally_mouse_name = "ROG Ally Mouse"; + +/* Changes to ally_drvdata must lock */ +static DEFINE_MUTEX(ally_data_mutex); +static struct ally_handheld ally_drvdata = { + .cfg_hdev = NULL, + .led_rgb_dev = NULL, + .ally_x_input = NULL, +}; + +static inline int asus_dev_set_report(struct hid_device *hdev, const u8 *buf, size_t len) +{ + unsigned char *dmabuf; + int ret; + + dmabuf = kmemdup(buf, len, GFP_KERNEL); + if (!dmabuf) + return -ENOMEM; + + ret = hid_hw_raw_request(hdev, buf[0], dmabuf, len, HID_FEATURE_REPORT, + HID_REQ_SET_REPORT); + kfree(dmabuf); + + return ret; +} + +static inline int asus_dev_get_report(struct hid_device *hdev, u8 *out, size_t len) +{ + return hid_hw_raw_request(hdev, HID_ALLY_GET_REPORT_ID, out, len, + HID_FEATURE_REPORT, HID_REQ_GET_REPORT); +} + +/** + * ally_gamepad_send_packet - Send a raw packet to the gamepad device. + * + * @ally: ally handheld structure + * @hdev: hid device + * @buf: Buffer containing the packet data + * @len: Length of data to send + * + * Return: count of data transferred, negative if error + */ +int ally_gamepad_send_packet(struct ally_handheld *ally, + struct hid_device *hdev, const u8 *buf, size_t len) +{ + int ret; + + mutex_lock(&ally->intf_mutex); + ret = asus_dev_set_report(hdev, buf, len); + mutex_unlock(&ally->intf_mutex); + + return ret; +} + +/** + * ally_gamepad_send_receive_packet - Send packet and receive response. + * + * @ally: ally handheld structure + * @hdev: hid device + * @buf: Buffer containing the packet data to send and receive response in + * @len: Length of buffer + * + * Return: count of data transferred, negative if error + */ +int ally_gamepad_send_receive_packet(struct ally_handheld *ally, + struct hid_device *hdev, u8 *buf, + size_t len) +{ + int ret; + + mutex_lock(&ally->intf_mutex); + ret = asus_dev_set_report(hdev, buf, len); + if (ret >= 0) { + memset(buf, 0, len); + ret = asus_dev_get_report(hdev, buf, len); + } + mutex_unlock(&ally->intf_mutex); + + return ret; +} + +/** + * ally_gamepad_send_one_byte_packet - Send a one-byte payload packet. + * + * @ally: ally handheld structure + * @hdev: hid device + * @command: Command code + * @param: Parameter byte + * + * Return: count of data transferred, negative if error + */ +int ally_gamepad_send_one_byte_packet(struct ally_handheld *ally, + struct hid_device *hdev, + enum ally_command_codes command, u8 param) +{ + u8 *packet; + int ret; + + packet = kzalloc(HID_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!packet) + return -ENOMEM; + + packet[0] = HID_ALLY_SET_REPORT_ID; + packet[1] = HID_ALLY_FEATURE_CODE_PAGE; + packet[2] = command; + packet[3] = 0x01; /* Length */ + packet[4] = param; + + ret = ally_gamepad_send_packet(ally, hdev, packet, + HID_ALLY_REPORT_SIZE); + kfree(packet); + return ret; +} + +/** + * ally_gamepad_send_two_byte_packet - Send a two-byte payload packet. + * + * @ally: ally handheld structure + * @hdev: hid device + * @command: Command code + * @param1: First parameter byte + * @param2: Second parameter byte + * + * Return: count of data transferred, negative if error + */ +int ally_gamepad_send_two_byte_packet(struct ally_handheld *ally, + struct hid_device *hdev, + enum ally_command_codes command, + u8 param1, u8 param2) +{ + u8 *packet; + int ret; + + packet = kzalloc(HID_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!packet) + return -ENOMEM; + + packet[0] = HID_ALLY_SET_REPORT_ID; + packet[1] = HID_ALLY_FEATURE_CODE_PAGE; + packet[2] = command; + packet[3] = 0x02; /* Length */ + packet[4] = param1; + packet[5] = param2; + + ret = ally_gamepad_send_packet(ally, hdev, packet, + HID_ALLY_REPORT_SIZE); + kfree(packet); + return ret; +} + +/* + * This should be called before any remapping attempts, and on driver init/resume. + */ +int ally_gamepad_check_ready(struct hid_device *hdev) +{ + int ret, count; + u8 *hidbuf; + struct ally_handheld *ally = hid_get_drvdata(hdev); + + hidbuf = kzalloc(HID_ALLY_REPORT_SIZE, GFP_KERNEL); + if (!hidbuf) + return -ENOMEM; + + ret = 0; + for (count = 0; count < READY_MAX_TRIES; count++) { + hidbuf[0] = HID_ALLY_SET_REPORT_ID; + hidbuf[1] = HID_ALLY_FEATURE_CODE_PAGE; + hidbuf[2] = CMD_CHECK_READY; + hidbuf[3] = 01; + + ret = ally_gamepad_send_receive_packet(ally, hdev, hidbuf, + HID_ALLY_REPORT_SIZE); + if (ret < 0) { + hid_err(hdev, "ROG Ally check failed: %d\n", ret); + continue; + } + + ret = hidbuf[2] == CMD_CHECK_READY; + if (ret) + break; + usleep_range(1000, 2000); + } + + if (count == READY_MAX_TRIES) + hid_warn(hdev, "ROG Ally never responded with a ready\n"); + + kfree(hidbuf); + return ret; +} + +u8 get_endpoint_address(struct hid_device *hdev) +{ + struct usb_host_endpoint *ep; + struct usb_interface *intf; + + intf = to_usb_interface(hdev->dev.parent); + if (!intf || !intf->cur_altsetting) + return -ENODEV; + + ep = intf->cur_altsetting->endpoint; + if (!ep) + return -ENODEV; + + return ep->desc.bEndpointAddress; +} + +/**************************************************************************************************/ +/* ROG Ally driver init */ +/**************************************************************************************************/ + +static int cad_sequence_state = 0; +static unsigned long cad_last_event_time = 0; + +/* Ally left buton emits a sequence of events: ctrl+alt+del. Capture this and emit only a single code */ +static bool handle_ctrl_alt_del(struct hid_device *hdev, u8 *data, int size) +{ + if (size < 16 || data[0] != 0x01) + return false; + + if (cad_sequence_state > 0 && time_after(jiffies, cad_last_event_time + msecs_to_jiffies(100))) + cad_sequence_state = 0; + + cad_last_event_time = jiffies; + + switch (cad_sequence_state) { + case 0: + if (data[1] == 0x01 && data[2] == 0x00 && data[3] == 0x00) { + cad_sequence_state = 1; + data[1] = 0x00; + return true; + } + break; + case 1: + if (data[1] == 0x05 && data[2] == 0x00 && data[3] == 0x00) { + cad_sequence_state = 2; + data[1] = 0x00; + return true; + } + break; + case 2: + if (data[1] == 0x05 && data[2] == 0x00 && data[3] == 0x4c) { + cad_sequence_state = 3; + data[1] = 0x00; + data[3] = 0x6F; // F20; + return true; + } + break; + case 3: + if (data[1] == 0x04 && data[2] == 0x00 && data[3] == 0x4c) { + cad_sequence_state = 4; + data[1] = 0x00; + data[1] = data[3] = 0x00; + return true; + } + break; + case 4: + if (data[1] == 0x00 && data[2] == 0x00 && data[3] == 0x4c) { + cad_sequence_state = 5; + data[3] = 0x00; + return true; + } + break; + } + cad_sequence_state = 0; + return false; +} + +static bool handle_ally_event(struct hid_device *hdev, u8 *data, int size) +{ + struct input_dev *keyboard_input; + int keycode = 0; + + if (data[0] == 0x5A) { + switch (data[1]) { + case 0x38: + keycode = KEY_F19; + break; + case 0xA6: + keycode = KEY_F16; + break; + case 0xA7: + keycode = KEY_F17; + break; + case 0xA8: + keycode = KEY_F18; + break; + default: + return false; + } + + keyboard_input = ally_drvdata.keyboard_input; + if (keyboard_input) { + input_report_key(keyboard_input, keycode, 1); + input_sync(keyboard_input); + input_report_key(keyboard_input, keycode, 0); + input_sync(keyboard_input); + return true; + } + + memset(data, 0, size); + } + return false; +} + +static int ally_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, + int size) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_x_input *ally_x; + int ep; + + if (!ally) + return -ENODEV; + + ep = get_endpoint_address(hdev); + if (ep != HID_ALLY_INTF_CFG_IN && ep != HID_ALLY_X_INTF_IN && ep != HID_ALLY_KEYBOARD_INTF_IN) + return 0; + + ally_x = ally->ally_x_input; + if (ally_x) { + if ((hdev->bus == BUS_USB && report->id == HID_ALLY_X_INPUT_REPORT && + size == HID_ALLY_X_INPUT_REPORT_SIZE) || + (data[0] == 0x5A)) { + if (ally_x_raw_event(ally_x, report, data, size)) + return 0; + } + } + + switch (ep) { + case HID_ALLY_INTF_CFG_IN: + if (handle_ally_event(hdev, data, size)) + return 0; + break; + case HID_ALLY_KEYBOARD_INTF_IN: + if (handle_ctrl_alt_del(hdev, data, size)) + return 0; + break; + } + + return 0; +} + +static int ally_hid_init(struct hid_device *hdev) +{ + int ret; + struct ally_handheld *ally = hid_get_drvdata(hdev); + + ret = ally_gamepad_send_packet(ally, hdev, EC_INIT_STRING, sizeof(EC_INIT_STRING)); + if (ret < 0) { + hid_err(hdev, "Ally failed to send init command: %d\n", ret); + goto cleanup; + } + + /* All gamepad configuration commands must go after the ally_gamepad_check_ready() */ + ret = ally_gamepad_check_ready(hdev); + if (ret < 0) + goto cleanup; + + ret = ally_gamepad_send_packet(ally, hdev, FORCE_FEEDBACK_OFF, sizeof(FORCE_FEEDBACK_OFF)); + if (ret < 0) + hid_err(hdev, "Ally failed to init force-feedback off: %d\n", ret); + +cleanup: + return ret; +} + +static int ally_input_configured(struct hid_device *hdev, struct hid_input *hi) +{ + int ep = get_endpoint_address(hdev); + + hid_info(hdev, "Input configured: endpoint 0x%02x, name: %s\n", ep, hi->input->name); + + if (ep == HID_ALLY_KEYBOARD_INTF_IN) + hi->input->name = ally_keyboard_name; + + if (ep == HID_ALLY_MOUSE_INTF_IN) + hi->input->name = ally_mouse_name; + + return 0; +} + +static int ally_hid_probe(struct hid_device *hdev, const struct hid_device_id *_id) +{ + int ret, ep; + + ep = get_endpoint_address(hdev); + if (ep < 0) + return ep; + + /*** CRITICAL START ***/ + mutex_lock(&ally_data_mutex); + if (ep == HID_ALLY_INTF_CFG_IN) + ally_drvdata.cfg_hdev = hdev; + if (ep == HID_ALLY_KEYBOARD_INTF_IN) + ally_drvdata.keyboard_hdev = hdev; + mutex_unlock(&ally_data_mutex); + /*** CRITICAL END ***/ + + hid_set_drvdata(hdev, &ally_drvdata); + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "Parse failed\n"); + return ret; + } + + if (ep == HID_ALLY_INTF_CFG_IN || ep == HID_ALLY_X_INTF_IN) { + ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW); + } else if (ep == HID_ALLY_KEYBOARD_INTF_IN) { + ret = hid_hw_start(hdev, HID_CONNECT_HIDINPUT | HID_CONNECT_HIDRAW); + if (!list_empty(&hdev->inputs)) { + struct hid_input *hidinput = list_first_entry(&hdev->inputs, struct hid_input, list); + ally_drvdata.keyboard_input = hidinput->input; + } + hid_info(hdev, "Connected keyboard interface with input events\n"); + } else { + ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); + hid_info(hdev, "Passing through HID events for endpoint: 0x%02x\n", ep); + return 0; + } + + ret = hid_hw_open(hdev); + if (ret) { + hid_err(hdev, "Failed to open HID device\n"); + goto err_stop; + } + + /* Initialize MCU even before alloc */ + ret = ally_hid_init(hdev); + if (ret < 0) + goto err_close; + + if (ep == HID_ALLY_INTF_CFG_IN) { + ret = ally_config_create(hdev, &ally_drvdata); + if (ret < 0) + hid_err(hdev, "Failed to create Ally configuration interface.\n"); + else + hid_info(hdev, "Created Ally configuration interface.\n"); + + ret = ally_rgb_create(hdev, &ally_drvdata); + if (ret < 0) + hid_err(hdev, "Failed to create Ally gamepad LEDs.\n"); + else + hid_info(hdev, "Created Ally RGB LED controls.\n"); + } + + if (ep == HID_ALLY_X_INTF_IN) { + ret = ally_x_create(hdev, &ally_drvdata); + if (ret < 0) { + hid_err(hdev, "Failed to create Ally X gamepad device.\n"); + ally_drvdata.ally_x_input = NULL; + goto err_close; + } else { + hid_info(hdev, "Created Ally X gamepad device.\n"); + } + // Not required since we send this inputs ep through the gamepad input dev + // if (drvdata.gamepad_cfg && drvdata.gamepad_cfg->input) { + // input_unregister_device(drvdata.gamepad_cfg->input); + // hid_info(hdev, "Ally X removed unrequired input dev.\n"); + // } + } + + return 0; + +err_close: + hid_hw_close(hdev); +err_stop: + hid_hw_stop(hdev); + return ret; +} + +static void ally_hid_remove(struct hid_device *hdev) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + + if (!ally) + goto out; + + if (ally->led_rgb_dev) + ally_rgb_remove(hdev, ally); + + if (ally->config) + ally_config_remove(hdev, ally); + + if (ally->ally_x_input) + ally_x_remove(hdev, ally); + +out: + hid_hw_close(hdev); + hid_hw_stop(hdev); +} + +static int ally_hid_reset_resume(struct hid_device *hdev) +{ + struct ally_handheld *ally = hid_get_drvdata(hdev); + int ret; + + if (!ally) + return -EINVAL; + + int ep = get_endpoint_address(hdev); + if (ep != HID_ALLY_INTF_CFG_IN) + return 0; + + ret = ally_hid_init(hdev); + if (ret < 0) + return ret; + + ally_rgb_resume(ally); + + return 0; +} + +static int ally_pm_thaw(struct device *dev) +{ + struct hid_device *hdev = to_hid_device(dev); + + if (!hdev) + return -EINVAL; + + return ally_hid_reset_resume(hdev); +} + +static int ally_pm_prepare(struct device *dev) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + + if (ally->led_rgb_dev) { + ally_rgb_store_settings(ally); + } + + return 0; +} + +static const struct dev_pm_ops ally_pm_ops = { + .thaw = ally_pm_thaw, + .prepare = ally_pm_prepare, +}; + +MODULE_DEVICE_TABLE(hid, rog_ally_devices); + +static struct hid_driver rog_ally_cfg = { .name = "asus_rog_ally", + .id_table = rog_ally_devices, + .probe = ally_hid_probe, + .remove = ally_hid_remove, + .raw_event = ally_raw_event, + .input_configured = ally_input_configured, + /* ALLy 1 requires this to reset device state correctly */ + .reset_resume = ally_hid_reset_resume, + .driver = { + .pm = &ally_pm_ops, + } +}; + +static int __init rog_ally_init(void) +{ + mutex_init(&ally_drvdata.intf_mutex); + return hid_register_driver(&rog_ally_cfg); +} + +static void __exit rog_ally_exit(void) +{ + mutex_destroy(&ally_drvdata.intf_mutex); + hid_unregister_driver(&rog_ally_cfg); +} + +module_init(rog_ally_init); +module_exit(rog_ally_exit); + +MODULE_AUTHOR("Luke D. Jones"); +MODULE_DESCRIPTION("HID Driver for ASUS ROG Ally handeheld."); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/asus-ally-hid/asus-ally-hid-input.c b/drivers/hid/asus-ally-hid/asus-ally-hid-input.c new file mode 100644 index 000000000000..4fc848d67c23 --- /dev/null +++ b/drivers/hid/asus-ally-hid/asus-ally-hid-input.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2023 Luke Jones + */ + +#include "linux/delay.h" +#include "linux/input-event-codes.h" +#include +#include + +#include "asus-ally.h" + +struct ally_x_input_report { + uint16_t x, y; + uint16_t rx, ry; + uint16_t z, rz; + uint8_t buttons[4]; +} __packed; + +/* The hatswitch outputs integers, we use them to index this X|Y pair */ +static const int hat_values[][2] = { + { 0, 0 }, { 0, -1 }, { 1, -1 }, { 1, 0 }, { 1, 1 }, + { 0, 1 }, { -1, 1 }, { -1, 0 }, { -1, -1 }, +}; + +static void ally_x_work(struct work_struct *work) +{ + struct ally_x_input *ally_x = container_of(work, struct ally_x_input, output_worker); + struct ff_report *ff_report = NULL; + bool update_qam_chord = false; + bool update_ff = false; + unsigned long flags; + + spin_lock_irqsave(&ally_x->lock, flags); + update_qam_chord = ally_x->update_qam_chord; + + update_ff = ally_x->update_ff; + if (ally_x->update_ff) { + ff_report = kmemdup(ally_x->ff_packet, sizeof(*ally_x->ff_packet), GFP_KERNEL); + ally_x->update_ff = false; + } + spin_unlock_irqrestore(&ally_x->lock, flags); + + if (update_ff && ff_report) { + ff_report->ff.magnitude_left = ff_report->ff.magnitude_strong; + ff_report->ff.magnitude_right = ff_report->ff.magnitude_weak; + ally_gamepad_send_packet(ally_x->ally, ally_x->hdev, + (u8 *)ff_report, sizeof(*ff_report)); + } + kfree(ff_report); + + if (update_qam_chord) { + /* + * The sleeps here are required to allow steam to register the button combo. + */ + input_report_key(ally_x->input, BTN_MODE, 1); + input_sync(ally_x->input); + msleep(150); + input_report_key(ally_x->input, BTN_A, 1); + input_sync(ally_x->input); + input_report_key(ally_x->input, BTN_A, 0); + input_sync(ally_x->input); + input_report_key(ally_x->input, BTN_MODE, 0); + input_sync(ally_x->input); + + spin_lock_irqsave(&ally_x->lock, flags); + ally_x->update_qam_chord = false; + spin_unlock_irqrestore(&ally_x->lock, flags); + } +} + +static int ally_x_play_effect(struct input_dev *idev, void *data, struct ff_effect *effect) +{ + struct hid_device *hdev = input_get_drvdata(idev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_x_input *ally_x = ally->ally_x_input; + unsigned long flags; + + if (effect->type != FF_RUMBLE) + return 0; + + spin_lock_irqsave(&ally_x->lock, flags); + ally_x->ff_packet->ff.magnitude_strong = effect->u.rumble.strong_magnitude / 512; + ally_x->ff_packet->ff.magnitude_weak = effect->u.rumble.weak_magnitude / 512; + ally_x->update_ff = true; + spin_unlock_irqrestore(&ally_x->lock, flags); + + if (ally_x->output_worker_initialized) + schedule_work(&ally_x->output_worker); + + return 0; +} + +/* Return true if event was handled, otherwise false */ +bool ally_x_raw_event(struct ally_x_input *ally_x, struct hid_report *report, u8 *data, + int size) +{ + struct ally_x_input_report *in_report; + unsigned long flags; + u8 byte; + + if (data[0] == 0x0B) { + in_report = (struct ally_x_input_report *)&data[1]; + + input_report_abs(ally_x->input, ABS_X, in_report->x - 32768); + input_report_abs(ally_x->input, ABS_Y, in_report->y - 32768); + input_report_abs(ally_x->input, ABS_RX, in_report->rx - 32768); + input_report_abs(ally_x->input, ABS_RY, in_report->ry - 32768); + input_report_abs(ally_x->input, ABS_Z, in_report->z); + input_report_abs(ally_x->input, ABS_RZ, in_report->rz); + + byte = in_report->buttons[0]; + input_report_key(ally_x->input, BTN_A, byte & BIT(0)); + input_report_key(ally_x->input, BTN_B, byte & BIT(1)); + input_report_key(ally_x->input, BTN_X, byte & BIT(2)); + input_report_key(ally_x->input, BTN_Y, byte & BIT(3)); + input_report_key(ally_x->input, BTN_TL, byte & BIT(4)); + input_report_key(ally_x->input, BTN_TR, byte & BIT(5)); + input_report_key(ally_x->input, BTN_SELECT, byte & BIT(6)); + input_report_key(ally_x->input, BTN_START, byte & BIT(7)); + + byte = in_report->buttons[1]; + input_report_key(ally_x->input, BTN_THUMBL, byte & BIT(0)); + input_report_key(ally_x->input, BTN_THUMBR, byte & BIT(1)); + input_report_key(ally_x->input, BTN_MODE, byte & BIT(2)); + + byte = in_report->buttons[2]; + input_report_abs(ally_x->input, ABS_HAT0X, hat_values[byte][0]); + input_report_abs(ally_x->input, ABS_HAT0Y, hat_values[byte][1]); + input_sync(ally_x->input); + + return true; + } + /* + * The MCU used on Ally provides many devices: gamepad, keyboord, mouse, other. + * The AC and QAM buttons route through another interface making it difficult to + * use the events unless we grab those and use them here. Only works for Ally X. + */ + else if (data[0] == 0x5A) { + if (ally_x->qam_mode) { + spin_lock_irqsave(&ally_x->lock, flags); + /* Right Armoury Crate button */ + if (data[1] == 0x38 && !ally_x->update_qam_chord) { + ally_x->update_qam_chord = true; + if (ally_x->output_worker_initialized) + schedule_work(&ally_x->output_worker); + } + spin_unlock_irqrestore(&ally_x->lock, flags); + /* Left/XBox button */ + input_report_key(ally_x->input, BTN_MODE, data[1] == 0xA6); + } else { + /* Right Armoury Crate button */ + input_report_key(ally_x->input, KEY_PROG1, data[1] == 0x38); + /* Left/XBox button */ + input_report_key(ally_x->input, KEY_F16, data[1] == 0xA6); + } + /* QAM long press */ + input_report_key(ally_x->input, KEY_F17, data[1] == 0xA7); + /* QAM long press released */ + input_report_key(ally_x->input, KEY_F18, data[1] == 0xA8); + input_sync(ally_x->input); + + return data[1] == 0xA6 || data[1] == 0xA7 || data[1] == 0xA8 || data[1] == 0x38; + } + + return false; +} + +static struct input_dev *ally_x_alloc_input_dev(struct hid_device *hdev, + const char *name_suffix) +{ + struct input_dev *input_dev; + + input_dev = devm_input_allocate_device(&hdev->dev); + if (!input_dev) + return ERR_PTR(-ENOMEM); + + input_dev->id.bustype = hdev->bus; + input_dev->id.vendor = hdev->vendor; + input_dev->id.product = hdev->product; + input_dev->id.version = hdev->version; + input_dev->uniq = hdev->uniq; + input_dev->name = "ASUS ROG Ally X Gamepad"; + + input_set_drvdata(input_dev, hdev); + + return input_dev; +} + +static ssize_t ally_x_qam_mode_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_x_input *ally_x = ally->ally_x_input; + + if (!ally_x) + return -ENODEV; + + return sysfs_emit(buf, "%d\n", ally_x->qam_mode); +} + +static ssize_t ally_x_qam_mode_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hid_device *hdev = to_hid_device(dev); + struct ally_handheld *ally = hid_get_drvdata(hdev); + struct ally_x_input *ally_x = ally->ally_x_input; + bool val; + int ret; + + if (!ally_x) + return -ENODEV; + + ret = kstrtobool(buf, &val); + if (ret < 0) + return ret; + + ally_x->qam_mode = val; + + return count; +} +ALLY_DEVICE_ATTR_RW(ally_x_qam_mode, qam_mode); + +static int ally_x_setup_input(struct hid_device *hdev, struct ally_x_input *ally_x) +{ + struct input_dev *input; + int ret; + + input = ally_x_alloc_input_dev(hdev, NULL); + if (IS_ERR(input)) + return PTR_ERR(input); + + input_set_abs_params(input, ABS_X, -32768, 32767, 0, 0); + input_set_abs_params(input, ABS_Y, -32768, 32767, 0, 0); + input_set_abs_params(input, ABS_RX, -32768, 32767, 0, 0); + input_set_abs_params(input, ABS_RY, -32768, 32767, 0, 0); + input_set_abs_params(input, ABS_Z, 0, 1023, 0, 0); + input_set_abs_params(input, ABS_RZ, 0, 1023, 0, 0); + input_set_abs_params(input, ABS_HAT0X, -1, 1, 0, 0); + input_set_abs_params(input, ABS_HAT0Y, -1, 1, 0, 0); + input_set_capability(input, EV_KEY, BTN_A); + input_set_capability(input, EV_KEY, BTN_B); + input_set_capability(input, EV_KEY, BTN_X); + input_set_capability(input, EV_KEY, BTN_Y); + input_set_capability(input, EV_KEY, BTN_TL); + input_set_capability(input, EV_KEY, BTN_TR); + input_set_capability(input, EV_KEY, BTN_SELECT); + input_set_capability(input, EV_KEY, BTN_START); + input_set_capability(input, EV_KEY, BTN_MODE); + input_set_capability(input, EV_KEY, BTN_THUMBL); + input_set_capability(input, EV_KEY, BTN_THUMBR); + + input_set_capability(input, EV_KEY, KEY_PROG1); + input_set_capability(input, EV_KEY, KEY_F16); + input_set_capability(input, EV_KEY, KEY_F17); + input_set_capability(input, EV_KEY, KEY_F18); + input_set_capability(input, EV_KEY, BTN_TRIGGER_HAPPY); + input_set_capability(input, EV_KEY, BTN_TRIGGER_HAPPY1); + + input_set_capability(input, EV_FF, FF_RUMBLE); + input_ff_create_memless(input, NULL, ally_x_play_effect); + + ret = input_register_device(input); + if (ret) { + input_unregister_device(input); + return ret; + } + + ally_x->input = input; + + return 0; +} + +int ally_x_create(struct hid_device *hdev, struct ally_handheld *ally) +{ + uint8_t max_output_report_size; + struct ally_x_input *ally_x; + struct ff_report *ff_report; + int ret; + + ally_x = devm_kzalloc(&hdev->dev, sizeof(*ally_x), GFP_KERNEL); + if (!ally_x) + return -ENOMEM; + + ally_x->hdev = hdev; + ally_x->ally = ally; + ally->ally_x_input = ally_x; + + max_output_report_size = sizeof(struct ally_x_input_report); + + ret = ally_x_setup_input(hdev, ally_x); + if (ret) + goto free_ally_x; + + INIT_WORK(&ally_x->output_worker, ally_x_work); + spin_lock_init(&ally_x->lock); + ally_x->output_worker_initialized = true; + ally_x->qam_mode = + true; + + ff_report = devm_kzalloc(&hdev->dev, sizeof(*ff_report), GFP_KERNEL); + if (!ff_report) { + ret = -ENOMEM; + goto free_ally_x; + } + + /* None of these bytes will change for the FF command for now */ + ff_report->report_id = 0x0D; + ff_report->ff.enable = 0x0F; /* Enable all by default */ + ff_report->ff.pulse_sustain_10ms = 0xFF; /* Duration */ + ff_report->ff.pulse_release_10ms = 0x00; /* Start Delay */ + ff_report->ff.loop_count = 0xEB; /* Loop Count */ + ally_x->ff_packet = ff_report; + + if (sysfs_create_file(&hdev->dev.kobj, &dev_attr_ally_x_qam_mode.attr)) { + ret = -ENODEV; + goto unregister_input; + } + + hid_info(hdev, "Registered Ally X controller using %s\n", + dev_name(&ally_x->input->dev)); + + return 0; + +unregister_input: + input_unregister_device(ally_x->input); +free_ally_x: + devm_kfree(&hdev->dev, ally_x); + return ret; +} + +void ally_x_remove(struct hid_device *hdev, struct ally_handheld *ally) +{ + if (ally->ally_x_input) { + sysfs_remove_file(&hdev->dev.kobj, &dev_attr_ally_x_qam_mode.attr); + + if (ally->ally_x_input->output_worker_initialized) + cancel_work_sync(&ally->ally_x_input->output_worker); + + ally->ally_x_input = NULL; + } +} diff --git a/drivers/hid/asus-ally-hid/asus-ally-rgb.c b/drivers/hid/asus-ally-hid/asus-ally-rgb.c new file mode 100644 index 000000000000..22aec39a7634 --- /dev/null +++ b/drivers/hid/asus-ally-hid/asus-ally-rgb.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2025 Luke Jones + */ + +#include "asus-ally.h" +#include "linux/delay.h" + +static const u8 EC_MODE_LED_APPLY[] = { 0x5A, 0xB4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 }; +static const u8 EC_MODE_LED_SET[] = { 0x5A, 0xB5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 }; + +static struct ally_rgb_resume_data resume_data; + +static void ally_rgb_schedule_work(struct ally_rgb_dev *led) +{ + unsigned long flags; + + if (!led) + return; + + spin_lock_irqsave(&led->lock, flags); + if (!led->removed) + schedule_work(&led->work); + spin_unlock_irqrestore(&led->lock, flags); +} + +/* + * The RGB still has the basic 0-3 level brightness. Since the multicolour + * brightness is being used in place, set this to max + */ +static int ally_rgb_set_bright_base_max(struct hid_device *hdev, struct ally_handheld *ally) +{ + u8 buf[] = { HID_ALLY_SET_RGB_REPORT_ID, 0xba, 0xc5, 0xc4, 0x02 }; + + return ally_gamepad_send_packet(ally, hdev, buf, sizeof(buf)); +} + +static void ally_rgb_do_work(struct work_struct *work) +{ + struct ally_rgb_dev *led = container_of(work, struct ally_rgb_dev, work); + unsigned long flags; + int ret; + + bool update_needed = false; + u8 red[4], green[4], blue[4]; + const int data_size = 12; /* 4 RGB zones × 3 colors */ + + u8 buf[16] = { [0] = HID_ALLY_SET_REPORT_ID, + [1] = HID_ALLY_FEATURE_CODE_PAGE, + [2] = CMD_LED_CONTROL, + [3] = data_size }; + + if (!led || !led->hdev) + return; + + spin_lock_irqsave(&led->lock, flags); + if (led->removed) { + spin_unlock_irqrestore(&led->lock, flags); + return; + } + + if (led->update_rgb) { + memcpy(red, led->red, sizeof(red)); + memcpy(green, led->green, sizeof(green)); + memcpy(blue, led->blue, sizeof(blue)); + led->update_rgb = false; + update_needed = true; + } + spin_unlock_irqrestore(&led->lock, flags); + + if (!update_needed) + return; + + for (int i = 0; i < 4; i++) { + buf[5 + i * 3] = green[i]; + buf[6 + i * 3] = blue[i]; + buf[4 + i * 3] = red[i]; + } + + ret = ally_gamepad_send_packet(led->ally, led->hdev, buf, sizeof(buf)); + if (ret < 0) + hid_err(led->hdev, "Ally failed to set gamepad backlight: %d\n", + ret); +} + +static void ally_rgb_set(struct led_classdev *cdev, + enum led_brightness brightness) +{ + struct led_classdev_mc *mc_cdev; + struct ally_rgb_dev *led; + int intensity, bright; + unsigned long flags; + + mc_cdev = lcdev_to_mccdev(cdev); + if (!mc_cdev) + return; + + led = container_of(mc_cdev, struct ally_rgb_dev, led_rgb_dev); + if (!led) + return; + + led_mc_calc_color_components(mc_cdev, brightness); + + spin_lock_irqsave(&led->lock, flags); + + led->update_rgb = true; + bright = mc_cdev->led_cdev.brightness; + + for (int i = 0; i < 4; i++) { + intensity = mc_cdev->subled_info[i].intensity; + led->red[i] = (((intensity >> 16) & 0xFF) * bright) / 255; + led->green[i] = (((intensity >> 8) & 0xFF) * bright) / 255; + led->blue[i] = ((intensity & 0xFF) * bright) / 255; + } + + resume_data.initialized = true; + + spin_unlock_irqrestore(&led->lock, flags); + + ally_rgb_schedule_work(led); +} + +static int ally_rgb_set_static_from_multi(struct hid_device *hdev, + struct ally_handheld *ally, u8 r, + u8 g, u8 b) +{ + u8 buf[17] = { HID_ALLY_SET_RGB_REPORT_ID, 0xb3 }; + int ret; + + /* + * Set single zone single colour based on the first LED of EC software mode. + * buf[2] = zone, buf[3] = mode + */ + buf[4] = r; + buf[5] = g; + buf[6] = b; + + ret = ally_gamepad_send_packet(ally, hdev, buf, sizeof(buf)); + if (ret < 0) + return ret; + + ret = ally_gamepad_send_packet(ally, hdev, EC_MODE_LED_APPLY, + sizeof(EC_MODE_LED_APPLY)); + if (ret < 0) + return ret; + + return ally_gamepad_send_packet(ally, hdev, EC_MODE_LED_SET, + sizeof(EC_MODE_LED_SET)); +} + +/* + * Store the RGB values for restoring on resume, and set the static mode to the first LED colour +*/ +void ally_rgb_store_settings(struct ally_handheld *ally) +{ + struct ally_rgb_dev *led_rgb; + int arr_size; + u8 r = 0, g = 0, b = 0; + + led_rgb = ally->led_rgb_dev; + if (!led_rgb || !led_rgb->hdev) + return; + + arr_size = sizeof(resume_data.red); + + /* Take a snapshot of current settings with locking */ + spin_lock_irq(&led_rgb->lock); + resume_data.brightness = led_rgb->led_rgb_dev.led_cdev.brightness; + memcpy(resume_data.red, led_rgb->red, arr_size); + memcpy(resume_data.green, led_rgb->green, arr_size); + memcpy(resume_data.blue, led_rgb->blue, arr_size); + r = resume_data.red[0]; + g = resume_data.green[0]; + b = resume_data.blue[0]; + spin_unlock_irq(&led_rgb->lock); + + ally_rgb_set_static_from_multi(led_rgb->hdev, ally, r, g, b); +} + +static void ally_rgb_restore_settings(struct ally_handheld *ally, + struct led_classdev *led_cdev, + struct mc_subled *mc_led_info) +{ + struct ally_rgb_dev *led_rgb_dev; + unsigned long flags; + int arr_size; + + led_rgb_dev = ally->led_rgb_dev; + if (!led_rgb_dev) + return; + + arr_size = sizeof(resume_data.red); + + spin_lock_irqsave(&led_rgb_dev->lock, flags); + + memcpy(led_rgb_dev->red, resume_data.red, arr_size); + memcpy(led_rgb_dev->green, resume_data.green, arr_size); + memcpy(led_rgb_dev->blue, resume_data.blue, arr_size); + + for (int i = 0; i < 4; i++) { + mc_led_info[i].intensity = (resume_data.red[i] << 16) | + (resume_data.green[i] << 8) | + resume_data.blue[i]; + } + led_cdev->brightness = resume_data.brightness; + + spin_unlock_irqrestore(&led_rgb_dev->lock, flags); +} + +/* Set LEDs. Call after any setup. */ +void ally_rgb_resume(struct ally_handheld *ally) +{ + struct ally_rgb_dev *led_rgb; + struct led_classdev *led_cdev; + struct mc_subled *mc_led_info; + + led_rgb = ally->led_rgb_dev; + if (!led_rgb) + return; + + led_cdev = &led_rgb->led_rgb_dev.led_cdev; + mc_led_info = led_rgb->led_rgb_dev.subled_info; + if (!led_cdev || !mc_led_info) + return; + + if (resume_data.initialized) { + ally_rgb_restore_settings(ally, led_cdev, mc_led_info); + + spin_lock_irq(&led_rgb->lock); + led_rgb->update_rgb = true; + spin_unlock_irq(&led_rgb->lock); + + ally_rgb_schedule_work(led_rgb); + ally_rgb_set_bright_base_max(led_rgb->hdev, ally); + } +} + +static int ally_rgb_register(struct hid_device *hdev, + struct ally_rgb_dev *led_rgb) +{ + struct mc_subled *mc_led_info; + struct led_classdev *led_cdev; + int ret; + + if (!hdev || !led_rgb) + return -EINVAL; + + mc_led_info = devm_kmalloc_array(&hdev->dev, 4, sizeof(*mc_led_info), + GFP_KERNEL | __GFP_ZERO); + if (!mc_led_info) + return -ENOMEM; + + for (int i = 0; i < 4; i++) { + mc_led_info[i].color_index = LED_COLOR_ID_RGB; + } + + led_rgb->led_rgb_dev.subled_info = mc_led_info; + led_rgb->led_rgb_dev.num_colors = 4; + + led_cdev = &led_rgb->led_rgb_dev.led_cdev; + led_cdev->brightness = 128; + led_cdev->name = "ally:rgb:joystick_rings"; + led_cdev->max_brightness = 255; + led_cdev->brightness_set = ally_rgb_set; + + ret = devm_led_classdev_multicolor_register(&hdev->dev, + &led_rgb->led_rgb_dev); + if (ret < 0) + hid_err(hdev, "Failed to register RGB LED device: %d\n", ret); + + return ret; +} + +int ally_rgb_create(struct hid_device *hdev, struct ally_handheld *ally) +{ + struct ally_rgb_dev *led_rgb; + int ret; + + led_rgb = devm_kzalloc(&hdev->dev, sizeof(struct ally_rgb_dev), + GFP_KERNEL); + if (!led_rgb) + return -ENOMEM; + + led_rgb->ally = ally; + led_rgb->hdev = hdev; + led_rgb->removed = false; + INIT_WORK(&led_rgb->work, ally_rgb_do_work); + spin_lock_init(&led_rgb->lock); + + /* Set the pointer in ally structure BEFORE doing any operations that might use it */ + ally->led_rgb_dev = led_rgb; + + ret = ally_rgb_register(hdev, led_rgb); + if (ret < 0) { + hid_err(hdev, "Failed to register RGB LED device: %d\n", ret); + cancel_work_sync(&led_rgb->work); + ally->led_rgb_dev = NULL; /* Reset pointer on failure */ + devm_kfree(&hdev->dev, led_rgb); + return ret; + } + + led_rgb->output_worker_initialized = true; + + ret = ally_rgb_set_bright_base_max(hdev, ally); + if (ret < 0) + hid_warn(hdev, "Failed to set maximum base brightness: %d\n", + ret); + + if (resume_data.initialized) { + msleep(1500); + spin_lock_irq(&led_rgb->lock); + led_rgb->update_rgb = true; + spin_unlock_irq(&led_rgb->lock); + ally_rgb_schedule_work(led_rgb); + } + + return 0; +} + +void ally_rgb_remove(struct hid_device *hdev, struct ally_handheld *ally) +{ + struct ally_rgb_dev *led_rgb; + unsigned long flags; + int ep; + + ep = get_endpoint_address(hdev); + if (ep != HID_ALLY_INTF_CFG_IN) + return; + + led_rgb = ally->led_rgb_dev; + if (!led_rgb) + return; + + /* Mark as removed to prevent new work from being scheduled */ + spin_lock_irqsave(&led_rgb->lock, flags); + if (led_rgb->removed) { + spin_unlock_irqrestore(&led_rgb->lock, flags); + return; + } + led_rgb->removed = true; + led_rgb->output_worker_initialized = false; + spin_unlock_irqrestore(&led_rgb->lock, flags); + + cancel_work_sync(&led_rgb->work); + + devm_led_classdev_multicolor_unregister(&hdev->dev, + &led_rgb->led_rgb_dev); + + ally->led_rgb_dev = NULL; + + hid_info(hdev, "Removed Ally RGB interface"); +} diff --git a/drivers/hid/asus-ally-hid/asus-ally.h b/drivers/hid/asus-ally-hid/asus-ally.h new file mode 100644 index 000000000000..fd9c788a4a42 --- /dev/null +++ b/drivers/hid/asus-ally-hid/asus-ally.h @@ -0,0 +1,314 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * + * HID driver for Asus ROG laptops and Ally + * + * Copyright (c) 2023 Luke Jones + */ + + #ifndef __ASUS_ALLY_H + #define __ASUS_ALLY_H + +#include +#include +#include + +#define HID_ALLY_KEYBOARD_INTF_IN 0x81 +#define HID_ALLY_MOUSE_INTF_IN 0x82 +#define HID_ALLY_INTF_CFG_IN 0x83 +#define HID_ALLY_X_INTF_IN 0x87 + +#define HID_ALLY_REPORT_SIZE 64 +#define HID_ALLY_GET_REPORT_ID 0x0D +#define HID_ALLY_SET_REPORT_ID 0x5A +#define HID_ALLY_SET_RGB_REPORT_ID 0x5D +#define HID_ALLY_FEATURE_CODE_PAGE 0xD1 + +#define HID_ALLY_X_INPUT_REPORT 0x0B +#define HID_ALLY_X_INPUT_REPORT_SIZE 16 + +enum ally_command_codes { + CMD_SET_GAMEPAD_MODE = 0x01, + CMD_SET_MAPPING = 0x02, + CMD_SET_JOYSTICK_MAPPING = 0x03, + CMD_SET_JOYSTICK_DEADZONE = 0x04, + CMD_SET_TRIGGER_RANGE = 0x05, + CMD_SET_VIBRATION_INTENSITY = 0x06, + CMD_LED_CONTROL = 0x08, + CMD_CHECK_READY = 0x0A, + CMD_SET_XBOX_CONTROLLER = 0x0B, + CMD_CHECK_XBOX_SUPPORT = 0x0C, + CMD_USER_CAL_DATA = 0x0D, + CMD_CHECK_USER_CAL_SUPPORT = 0x0E, + CMD_SET_TURBO_PARAMS = 0x0F, + CMD_CHECK_TURBO_SUPPORT = 0x10, + CMD_CHECK_RESP_CURVE_SUPPORT = 0x12, + CMD_SET_RESP_CURVE = 0x13, + CMD_CHECK_DIR_TO_BTN_SUPPORT = 0x14, + CMD_SET_GYRO_PARAMS = 0x15, + CMD_CHECK_GYRO_TO_JOYSTICK = 0x16, + CMD_CHECK_ANTI_DEADZONE = 0x17, + CMD_SET_ANTI_DEADZONE = 0x18, +}; + +enum ally_gamepad_mode { + ALLY_GAMEPAD_MODE_GAMEPAD = 0x01, + ALLY_GAMEPAD_MODE_KEYBOARD = 0x02, +}; + +static const char *const gamepad_mode_names[] = { + [ALLY_GAMEPAD_MODE_GAMEPAD] = "gamepad", + [ALLY_GAMEPAD_MODE_KEYBOARD] = "keyboard" +}; + +/* Button identifiers for the attribute system */ +enum ally_button_id { + ALLY_BTN_A, + ALLY_BTN_B, + ALLY_BTN_X, + ALLY_BTN_Y, + ALLY_BTN_LB, + ALLY_BTN_RB, + ALLY_BTN_DU, + ALLY_BTN_DD, + ALLY_BTN_DL, + ALLY_BTN_DR, + ALLY_BTN_J0B, + ALLY_BTN_J1B, + ALLY_BTN_MENU, + ALLY_BTN_VIEW, + ALLY_BTN_M1, + ALLY_BTN_M2, + ALLY_BTN_MAX +}; + +/* Names for the button directories in sysfs */ +static const char *const ally_button_names[ALLY_BTN_MAX] = { + [ALLY_BTN_A] = "btn_a", + [ALLY_BTN_B] = "btn_b", + [ALLY_BTN_X] = "btn_x", + [ALLY_BTN_Y] = "btn_y", + [ALLY_BTN_LB] = "btn_lb", + [ALLY_BTN_RB] = "btn_rb", + [ALLY_BTN_DU] = "dpad_up", + [ALLY_BTN_DD] = "dpad_down", + [ALLY_BTN_DL] = "dpad_left", + [ALLY_BTN_DR] = "dpad_right", + [ALLY_BTN_J0B] = "btn_l3", + [ALLY_BTN_J1B] = "btn_r3", + [ALLY_BTN_MENU] = "btn_menu", + [ALLY_BTN_VIEW] = "btn_view", + [ALLY_BTN_M1] = "btn_m1", + [ALLY_BTN_M2] = "btn_m2", +}; + +struct ally_rgb_resume_data { + uint8_t brightness; + uint8_t red[4]; + uint8_t green[4]; + uint8_t blue[4]; + bool initialized; +}; + +struct ally_rgb_dev { + struct ally_handheld *ally; + struct hid_device *hdev; + struct led_classdev_mc led_rgb_dev; + struct work_struct work; + bool output_worker_initialized; + spinlock_t lock; + + bool removed; + bool update_rgb; + uint8_t red[4]; + uint8_t green[4]; + uint8_t blue[4]; +}; + +/* rumble packet structure */ +struct ff_data { + u8 enable; + u8 magnitude_left; + u8 magnitude_right; + u8 magnitude_strong; + u8 magnitude_weak; + u8 pulse_sustain_10ms; + u8 pulse_release_10ms; + u8 loop_count; +} __packed; + +struct ff_report { + u8 report_id; + struct ff_data ff; +} __packed; + +struct ally_x_input { + struct ally_handheld *ally; + struct input_dev *input; + struct hid_device *hdev; + spinlock_t lock; + + struct work_struct output_worker; + bool output_worker_initialized; + + /* Set if the left QAM emits Guide/Mode and right QAM emits Home + A chord */ + bool qam_mode; + /* Prevent multiple queued event due to the enforced delay in worker */ + bool update_qam_chord; + + struct ff_report *ff_packet; + bool update_ff; +}; + +struct resp_curve_param { + u8 move; + u8 resp; +} __packed; + +struct joystick_resp_curve { + struct resp_curve_param entry_1; + struct resp_curve_param entry_2; + struct resp_curve_param entry_3; + struct resp_curve_param entry_4; +} __packed; + +/* + * Button turbo parameters structure + * Each button can have: + * - turbo: Turbo press interval in multiple of 50ms (0 = disabled, 1-20 = 50ms-1000ms) + * - toggle: Toggle interval (0 = disabled) + */ +struct button_turbo_params { + u8 turbo; + u8 toggle; +} __packed; + +/* Collection of all button turbo settings */ +struct turbo_config { + struct button_turbo_params btn_du; /* D-pad Up */ + struct button_turbo_params btn_dd; /* D-pad Down */ + struct button_turbo_params btn_dl; /* D-pad Left */ + struct button_turbo_params btn_dr; /* D-pad Right */ + struct button_turbo_params btn_j0b; /* Left joystick button */ + struct button_turbo_params btn_j1b; /* Right joystick button */ + struct button_turbo_params btn_lb; /* Left bumper */ + struct button_turbo_params btn_rb; /* Right bumper */ + struct button_turbo_params btn_a; /* A button */ + struct button_turbo_params btn_b; /* B button */ + struct button_turbo_params btn_x; /* X button */ + struct button_turbo_params btn_y; /* Y button */ + struct button_turbo_params btn_view; /* View button */ + struct button_turbo_params btn_menu; /* Menu button */ + struct button_turbo_params btn_m2; /* M2 button */ + struct button_turbo_params btn_m1; /* M1 button */ +}; + +struct ally_config { + struct hid_device *hdev; + /* Must be locked if the data is being changed */ + struct mutex config_mutex; + bool initialized; + + /* Device capabilities flags */ + bool is_ally_x; + bool xbox_controller_support; + bool user_cal_support; + bool turbo_support; + bool resp_curve_support; + bool dir_to_btn_support; + bool gyro_support; + bool anti_deadzone_support; + + /* Current settings */ + bool xbox_controller_enabled; + u8 gamepad_mode; + u8 left_deadzone; + u8 left_outer_threshold; + u8 right_deadzone; + u8 right_outer_threshold; + u8 left_anti_deadzone; + u8 right_anti_deadzone; + u8 left_trigger_min; + u8 left_trigger_max; + u8 right_trigger_min; + u8 right_trigger_max; + + /* Vibration settings */ + u8 vibration_intensity_left; + u8 vibration_intensity_right; + bool vibration_active; + + struct turbo_config turbo; + struct button_sysfs_entry *button_entries; + void *button_mappings; /* ally_button_mapping array indexed by gamepad_mode */ + + struct joystick_resp_curve left_curve; + struct joystick_resp_curve right_curve; +}; + +struct ally_handheld { + /* All read/write to IN interfaces must lock */ + struct mutex intf_mutex; + struct hid_device *cfg_hdev; + + struct ally_rgb_dev *led_rgb_dev; + + struct ally_x_input *ally_x_input; + + struct hid_device *keyboard_hdev; + struct input_dev *keyboard_input; + + struct ally_config *config; +}; + +int ally_gamepad_send_packet(struct ally_handheld *ally, + struct hid_device *hdev, const u8 *buf, + size_t len); +int ally_gamepad_send_receive_packet(struct ally_handheld *ally, + struct hid_device *hdev, u8 *buf, + size_t len); +int ally_gamepad_send_one_byte_packet(struct ally_handheld *ally, + struct hid_device *hdev, + enum ally_command_codes command, + u8 param); +int ally_gamepad_send_two_byte_packet(struct ally_handheld *ally, + struct hid_device *hdev, + enum ally_command_codes command, + u8 param1, u8 param2); +int ally_gamepad_check_ready(struct hid_device *hdev); +u8 get_endpoint_address(struct hid_device *hdev); + +int ally_rgb_create(struct hid_device *hdev, struct ally_handheld *ally); +void ally_rgb_remove(struct hid_device *hdev, struct ally_handheld *ally); +void ally_rgb_store_settings(struct ally_handheld *ally); +void ally_rgb_resume(struct ally_handheld *ally); + +int ally_x_create(struct hid_device *hdev, struct ally_handheld *ally); +void ally_x_remove(struct hid_device *hdev, struct ally_handheld *ally); +bool ally_x_raw_event(struct ally_x_input *ally_x, struct hid_report *report, u8 *data, + int size); + +int ally_config_create(struct hid_device *hdev, struct ally_handheld *ally); +void ally_config_remove(struct hid_device *hdev, struct ally_handheld *ally); + +#define ALLY_DEVICE_ATTR_RW(_name, _sysfs_name) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_sysfs_name, 0644, _name##_show, _name##_store) + +#define ALLY_DEVICE_ATTR_RO(_name, _sysfs_name) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_sysfs_name, 0444, _name##_show, NULL) + +#define ALLY_DEVICE_ATTR_WO(_name, _sysfs_name) \ + struct device_attribute dev_attr_##_name = \ + __ATTR(_sysfs_name, 0200, NULL, _name##_store) + +#define ALLY_DEVICE_CONST_ATTR_RO(fname, sysfs_name, value) \ + static ssize_t fname##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ + { \ + return sprintf(buf, value); \ + } \ + struct device_attribute dev_attr_##fname = \ + __ATTR(sysfs_name, 0444, fname##_show, NULL) + +#endif /* __ASUS_ALLY_H */ diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 46e3e42f9eb5..52882d6589e1 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -52,6 +52,10 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define FEATURE_KBD_LED_REPORT_ID1 0x5d #define FEATURE_KBD_LED_REPORT_ID2 0x5e +#define ROG_ALLY_REPORT_SIZE 64 +#define ROG_ALLY_X_MIN_MCU 313 +#define ROG_ALLY_MIN_MCU 319 + #define SUPPORT_KBD_BACKLIGHT BIT(0) #define MAX_TOUCH_MAJOR 8 @@ -84,6 +88,7 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad"); #define QUIRK_MEDION_E1239T BIT(10) #define QUIRK_ROG_NKEY_KEYBOARD BIT(11) #define QUIRK_ROG_CLAYMORE_II_KEYBOARD BIT(12) +#define QUIRK_ROG_ALLY_XPAD BIT(13) #define I2C_KEYBOARD_QUIRKS (QUIRK_FIX_NOTEBOOK_REPORT | \ QUIRK_NO_INIT_REPORTS | \ @@ -534,9 +539,102 @@ static bool asus_kbd_wmi_led_control_present(struct hid_device *hdev) return !!(value & ASUS_WMI_DSTS_PRESENCE_BIT); } +/* + * We don't care about any other part of the string except the version section. + * Example strings: FGA80100.RC72LA.312_T01, FGA80100.RC71LS.318_T01 + * The bytes "5a 05 03 31 00 1a 13" and possibly more come before the version + * string, and there may be additional bytes after the version string such as + * "75 00 74 00 65 00" or a postfix such as "_T01" + */ +static int mcu_parse_version_string(const u8 *response, size_t response_size) +{ + const u8 *end = response + response_size; + const u8 *p = response; + int dots, err, version; + char buf[4]; + + dots = 0; + while (p < end && dots < 2) { + if (*p++ == '.') + dots++; + } + + if (dots != 2 || p >= end || (p + 3) >= end) + return -EINVAL; + + memcpy(buf, p, 3); + buf[3] = '\0'; + + err = kstrtoint(buf, 10, &version); + if (err || version < 0) + return -EINVAL; + + return version; +} + +static int mcu_request_version(struct hid_device *hdev) +{ + u8 *response __free(kfree) = kzalloc(ROG_ALLY_REPORT_SIZE, GFP_KERNEL); + const u8 request[] = { 0x5a, 0x05, 0x03, 0x31, 0x00, 0x20 }; + int ret; + + if (!response) + return -ENOMEM; + + ret = asus_kbd_set_report(hdev, request, sizeof(request)); + if (ret < 0) + return ret; + + ret = hid_hw_raw_request(hdev, FEATURE_REPORT_ID, response, + ROG_ALLY_REPORT_SIZE, HID_FEATURE_REPORT, + HID_REQ_GET_REPORT); + if (ret < 0) + return ret; + + ret = mcu_parse_version_string(response, ROG_ALLY_REPORT_SIZE); + if (ret < 0) { + pr_err("Failed to parse MCU version: %d\n", ret); + print_hex_dump(KERN_ERR, "MCU: ", DUMP_PREFIX_NONE, + 16, 1, response, ROG_ALLY_REPORT_SIZE, false); + } + + return ret; +} + +static void validate_mcu_fw_version(struct hid_device *hdev, int idProduct) +{ + int min_version, version; + + version = mcu_request_version(hdev); + if (version < 0) + return; + + switch (idProduct) { + case USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY: + min_version = ROG_ALLY_MIN_MCU; + break; + case USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X: + min_version = ROG_ALLY_X_MIN_MCU; + break; + default: + min_version = 0; + } + + if (version < min_version) { + hid_warn(hdev, + "The MCU firmware version must be %d or greater to avoid issues with suspend.\n", + min_version); + } else { + set_ally_mcu_hack(ASUS_WMI_ALLY_MCU_HACK_DISABLED); + set_ally_mcu_powersave(true); + } +} + static int asus_kbd_register_leds(struct hid_device *hdev) { struct asus_drvdata *drvdata = hid_get_drvdata(hdev); + struct usb_interface *intf; + struct usb_device *udev; unsigned char kbd_func; int ret; @@ -560,6 +658,14 @@ static int asus_kbd_register_leds(struct hid_device *hdev) if (ret < 0) return ret; } + + if (drvdata->quirks & QUIRK_ROG_ALLY_XPAD) { + intf = to_usb_interface(hdev->dev.parent); + udev = interface_to_usbdev(intf); + validate_mcu_fw_version(hdev, + le16_to_cpu(udev->descriptor.idProduct)); + } + } else { /* Initialize keyboard */ ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); @@ -1278,12 +1384,17 @@ static const struct hid_device_id asus_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR), QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, + + /* asus-ally-hid driver takes over */ + #if !IS_REACHABLE(CONFIG_ASUS_ALLY_HID) { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD}, { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X), - QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD }, + QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_ROG_ALLY_XPAD }, + #endif /* !IS_REACHABLE(CONFIG_ASUS_ALLY_HID) */ + { HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD), QUIRK_ROG_CLAYMORE_II_KEYBOARD }, @@ -1327,4 +1438,5 @@ static struct hid_driver asus_driver = { }; module_hid_driver(asus_driver); +MODULE_IMPORT_NS("ASUS_WMI"); MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 1062731315a2..594cde034707 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -221,6 +221,7 @@ #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2 0x19b6 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3 0x1a30 #define USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR 0x18c6 +#define USB_DEVICE_ID_ASUSTEK_ROG_RAIKIRI_PAD 0x1abb #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY 0x1abe #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY_X 0x1b4c #define USB_DEVICE_ID_ASUSTEK_ROG_CLAYMORE_II_KEYBOARD 0x196b diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 43407e76476b..731d18308cb6 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -267,6 +267,18 @@ config ASUS_WIRELESS If you choose to compile this driver as a module the module will be called asus-wireless. +config ASUS_ARMOURY + tristate "ASUS Armoury driver" + depends on ASUS_WMI + select FW_ATTR_CLASS + help + Say Y here if you have a WMI aware Asus machine and would like to use the + firmware_attributes API to control various settings typically exposed in + the ASUS Armoury Crate application available on Windows. + + To compile this driver as a module, choose M here: the module will + be called asus-armoury. + config ASUS_WMI tristate "ASUS WMI Driver" depends on ACPI_WMI @@ -289,6 +301,17 @@ config ASUS_WMI To compile this driver as a module, choose M here: the module will be called asus-wmi. +config ASUS_WMI_DEPRECATED_ATTRS + bool "BIOS option support in WMI platform (DEPRECATED)" + depends on ASUS_WMI + default y + help + Say Y to expose the configurable BIOS options through the asus-wmi + driver. + + This can be used with or without the asus-armoury driver which + has the same attributes, but more, and better features. + config ASUS_NB_WMI tristate "Asus Notebook WMI Driver" depends on ASUS_WMI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index 650dfbebb6c8..3a0085f941d9 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_APPLE_GMUX) += apple-gmux.o # ASUS obj-$(CONFIG_ASUS_LAPTOP) += asus-laptop.o obj-$(CONFIG_ASUS_WIRELESS) += asus-wireless.o +obj-$(CONFIG_ASUS_ARMOURY) += asus-armoury.o obj-$(CONFIG_ASUS_WMI) += asus-wmi.o obj-$(CONFIG_ASUS_NB_WMI) += asus-nb-wmi.o obj-$(CONFIG_ASUS_TF103C_DOCK) += asus-tf103c-dock.o diff --git a/drivers/platform/x86/asus-armoury.c b/drivers/platform/x86/asus-armoury.c new file mode 100644 index 000000000000..84abc92bd365 --- /dev/null +++ b/drivers/platform/x86/asus-armoury.c @@ -0,0 +1,1202 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Asus Armoury (WMI) attributes driver. This driver uses the fw_attributes + * class to expose the various WMI functions that many gaming and some + * non-gaming ASUS laptops have available. + * These typically don't fit anywhere else in the sysfs such as under LED class, + * hwmon or other, and are set in Windows using the ASUS Armoury Crate tool. + * + * Copyright(C) 2024 Luke Jones + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asus-armoury.h" +#include "firmware_attributes_class.h" + +#define ASUS_NB_WMI_EVENT_GUID "0B3CBB35-E3C2-45ED-91C2-4C5A6D195D1C" + +#define ASUS_MINI_LED_MODE_MASK 0x03 +/* Standard modes for devices with only on/off */ +#define ASUS_MINI_LED_OFF 0x00 +#define ASUS_MINI_LED_ON 0x01 +/* Like "on" but the effect is more vibrant or brighter */ +#define ASUS_MINI_LED_STRONG_MODE 0x02 +/* New modes for devices with 3 mini-led mode types */ +#define ASUS_MINI_LED_2024_WEAK 0x00 +#define ASUS_MINI_LED_2024_STRONG 0x01 +#define ASUS_MINI_LED_2024_OFF 0x02 + +/* Power tunable attribute name defines */ +#define ATTR_PPT_PL1_SPL "ppt_pl1_spl" +#define ATTR_PPT_PL2_SPPT "ppt_pl2_sppt" +#define ATTR_PPT_PL3_FPPT "ppt_pl3_fppt" +#define ATTR_PPT_APU_SPPT "ppt_apu_sppt" +#define ATTR_PPT_PLATFORM_SPPT "ppt_platform_sppt" +#define ATTR_NV_DYNAMIC_BOOST "nv_dynamic_boost" +#define ATTR_NV_TEMP_TARGET "nv_temp_target" +#define ATTR_NV_BASE_TGP "nv_base_tgp" +#define ATTR_NV_TGP "nv_tgp" + +#define ASUS_POWER_CORE_MASK GENMASK(15, 8) +#define ASUS_PERF_CORE_MASK GENMASK(7, 0) + +enum cpu_core_type { + CPU_CORE_PERF = 0, + CPU_CORE_POWER, +}; + +enum cpu_core_value { + CPU_CORE_DEFAULT = 0, + CPU_CORE_MIN, + CPU_CORE_MAX, + CPU_CORE_CURRENT, +}; + +#define CPU_PERF_CORE_COUNT_MIN 4 +#define CPU_POWR_CORE_COUNT_MIN 0 + +/* Tunables provided by ASUS for gaming laptops */ +struct cpu_cores { + u32 cur_perf_cores; + u32 min_perf_cores; + u32 max_perf_cores; + u32 cur_power_cores; + u32 min_power_cores; + u32 max_power_cores; +}; + +struct rog_tunables { + const struct power_limits *power_limits; + u32 ppt_pl1_spl; // cpu + u32 ppt_pl2_sppt; // cpu + u32 ppt_pl3_fppt; // cpu + u32 ppt_apu_sppt; // plat + u32 ppt_platform_sppt; // plat + + u32 nv_dynamic_boost; + u32 nv_temp_target; + u32 nv_tgp; +}; + +static struct asus_armoury_priv { + struct device *fw_attr_dev; + struct kset *fw_attr_kset; + + struct cpu_cores *cpu_cores; + /* Index 0 for DC, 1 for AC */ + struct rog_tunables *rog_tunables[2]; + u32 mini_led_dev_id; + u32 gpu_mux_dev_id; + /* + * Mutex to prevent big/little core count changes writing to same + * endpoint at the same time. Must lock during attr store. + */ + struct mutex cpu_core_mutex; +} asus_armoury = { + .cpu_core_mutex = __MUTEX_INITIALIZER(asus_armoury.cpu_core_mutex) +}; + +struct fw_attrs_group { + bool pending_reboot; +}; + +static struct fw_attrs_group fw_attrs = { + .pending_reboot = false, +}; + +struct asus_attr_group { + const struct attribute_group *attr_group; + u32 wmi_devid; +}; + +static bool asus_wmi_is_present(u32 dev_id) +{ + u32 retval; + int status; + + status = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DSTS, dev_id, 0, &retval); + pr_debug("%s called (0x%08x), retval: 0x%08x\n", __func__, dev_id, retval); + + return status == 0 && (retval & ASUS_WMI_DSTS_PRESENCE_BIT); +} + +static void asus_set_reboot_and_signal_event(void) +{ + fw_attrs.pending_reboot = true; + kobject_uevent(&asus_armoury.fw_attr_dev->kobj, KOBJ_CHANGE); +} + +static ssize_t pending_reboot_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", fw_attrs.pending_reboot); +} + +static struct kobj_attribute pending_reboot = __ATTR_RO(pending_reboot); + +static bool asus_bios_requires_reboot(struct kobj_attribute *attr) +{ + return !strcmp(attr->attr.name, "gpu_mux_mode") || + !strcmp(attr->attr.name, "cores_performance") || + !strcmp(attr->attr.name, "cores_efficiency") || + !strcmp(attr->attr.name, "panel_hd_mode"); +} + +static int armoury_wmi_set_devstate(struct kobj_attribute *attr, u32 value, u32 wmi_dev) +{ + u32 result; + int err; + + err = asus_wmi_set_devstate(wmi_dev, value, &result); + if (err) { + pr_err("Failed to set %s: %d\n", attr->attr.name, err); + return err; + } + /* + * !1 is usually considered a fail by ASUS, but some WMI methods do use > 1 + * to return a status code or similar. + */ + if (result < 1) { + pr_err("Failed to set %s: (result): 0x%x\n", attr->attr.name, result); + return -EIO; + } + + return 0; +} + +/** + * attr_int_store() - Send an int to wmi method, checks if within min/max exclusive. + * @kobj: Pointer to the driver object. + * @attr: Pointer to the attribute calling this function. + * @buf: The buffer to read from, this is parsed to `int` type. + * @count: Required by sysfs attribute macros, pass in from the callee attr. + * @min: Minimum accepted value. Below this returns -EINVAL. + * @max: Maximum accepted value. Above this returns -EINVAL. + * @store_value: Pointer to where the parsed value should be stored. + * @wmi_dev: The WMI function ID to use. + * + * This function is intended to be generic so it can be called from any "_store" + * attribute which works only with integers. The integer to be sent to the WMI method + * is range checked and an error returned if out of range. + * + * If the value is valid and WMI is success, then the sysfs attribute is notified + * and if asus_bios_requires_reboot() is true then reboot attribute is also notified. + * + * Returns: Either count, or an error. + */ +static ssize_t attr_uint_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, + size_t count, u32 min, u32 max, u32 *store_value, u32 wmi_dev) +{ + u32 value; + int err; + + err = kstrtouint(buf, 10, &value); + if (err) + return err; + + if (value < min || value > max) + return -EINVAL; + + err = armoury_wmi_set_devstate(attr, value, wmi_dev); + if (err) + return err; + + if (store_value != NULL) + *store_value = value; + sysfs_notify(kobj, NULL, attr->attr.name); + + if (asus_bios_requires_reboot(attr)) + asus_set_reboot_and_signal_event(); + + return count; +} + +static ssize_t enum_type_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "enumeration\n"); +} + +static ssize_t int_type_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "integer\n"); +} + +/* Mini-LED mode **************************************************************/ +static ssize_t mini_led_mode_current_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + u32 value; + int err; + + err = asus_wmi_get_devstate_dsts(asus_armoury.mini_led_dev_id, &value); + if (err) + return err; + + value &= ASUS_MINI_LED_MODE_MASK; + + /* + * Remap the mode values to match previous generation mini-LED. The last gen + * WMI 0 == off, while on this version WMI 2 == off (flipped). + */ + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE2) { + switch (value) { + case ASUS_MINI_LED_2024_WEAK: + value = ASUS_MINI_LED_ON; + break; + case ASUS_MINI_LED_2024_STRONG: + value = ASUS_MINI_LED_STRONG_MODE; + break; + case ASUS_MINI_LED_2024_OFF: + value = ASUS_MINI_LED_OFF; + break; + } + } + + return sysfs_emit(buf, "%u\n", value); +} + +static ssize_t mini_led_mode_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + u32 mode; + int err; + + err = kstrtou32(buf, 10, &mode); + if (err) + return err; + + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE && + mode > ASUS_MINI_LED_ON) + return -EINVAL; + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE2 && + mode > ASUS_MINI_LED_STRONG_MODE) + return -EINVAL; + + /* + * Remap the mode values so expected behaviour is the same as the last + * generation of mini-LED with 0 == off, 1 == on. + */ + if (asus_armoury.mini_led_dev_id == ASUS_WMI_DEVID_MINI_LED_MODE2) { + switch (mode) { + case ASUS_MINI_LED_OFF: + mode = ASUS_MINI_LED_2024_OFF; + break; + case ASUS_MINI_LED_ON: + mode = ASUS_MINI_LED_2024_WEAK; + break; + case ASUS_MINI_LED_STRONG_MODE: + mode = ASUS_MINI_LED_2024_STRONG; + break; + } + } + + err = armoury_wmi_set_devstate(attr, mode, asus_armoury.mini_led_dev_id); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + + return count; +} + +static ssize_t mini_led_mode_possible_values_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + switch (asus_armoury.mini_led_dev_id) { + case ASUS_WMI_DEVID_MINI_LED_MODE: + return sysfs_emit(buf, "0;1\n"); + case ASUS_WMI_DEVID_MINI_LED_MODE2: + return sysfs_emit(buf, "0;1;2\n"); + } + + return sysfs_emit(buf, "0\n"); +} + +ATTR_GROUP_ENUM_CUSTOM(mini_led_mode, "mini_led_mode", "Set the mini-LED backlight mode"); + +static ssize_t gpu_mux_mode_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int result, err; + u32 optimus; + + err = kstrtou32(buf, 10, &optimus); + if (err) + return err; + + if (optimus > 1) + return -EINVAL; + + if (asus_wmi_is_present(ASUS_WMI_DEVID_DGPU)) { + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_DGPU, &result); + if (err) + return err; + if (result && !optimus) { + pr_warn("Can not switch MUX to dGPU mode when dGPU is disabled: %02X %02X\n", + result, optimus); + return -ENODEV; + } + } + + if (asus_wmi_is_present(ASUS_WMI_DEVID_EGPU)) { + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_EGPU, &result); + if (err) + return err; + if (result && !optimus) { + pr_warn("Can not switch MUX to dGPU mode when eGPU is enabled\n"); + return -ENODEV; + } + } + + err = armoury_wmi_set_devstate(attr, optimus, asus_armoury.gpu_mux_dev_id); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + asus_set_reboot_and_signal_event(); + + return count; +} +WMI_SHOW_INT(gpu_mux_mode_current_value, "%d\n", asus_armoury.gpu_mux_dev_id); +ATTR_GROUP_BOOL_CUSTOM(gpu_mux_mode, "gpu_mux_mode", "Set the GPU display MUX mode"); + +/* + * A user may be required to store the value twice, typical store first, then + * rescan PCI bus to activate power, then store a second time to save correctly. + */ +static ssize_t dgpu_disable_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int result, err; + u32 disable; + + err = kstrtou32(buf, 10, &disable); + if (err) + return err; + + if (disable > 1) + return -EINVAL; + + if (asus_armoury.gpu_mux_dev_id) { + err = asus_wmi_get_devstate_dsts(asus_armoury.gpu_mux_dev_id, &result); + if (err) + return err; + if (!result && disable) { + pr_warn("Can not disable dGPU when the MUX is in dGPU mode\n"); + return -ENODEV; + } + } + + err = armoury_wmi_set_devstate(attr, disable, ASUS_WMI_DEVID_DGPU); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + + return count; +} +WMI_SHOW_INT(dgpu_disable_current_value, "%d\n", ASUS_WMI_DEVID_DGPU); +ATTR_GROUP_BOOL_CUSTOM(dgpu_disable, "dgpu_disable", "Disable the dGPU"); + +/* The ACPI call to enable the eGPU also disables the internal dGPU */ +static ssize_t egpu_enable_current_value_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int result, err; + u32 enable; + + err = kstrtou32(buf, 10, &enable); + if (err) + return err; + + if (enable > 1) + return -EINVAL; + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_EGPU_CONNECTED, &result); + if (err) { + pr_warn("Failed to get eGPU connection status: %d\n", err); + return err; + } + + if (asus_armoury.gpu_mux_dev_id) { + err = asus_wmi_get_devstate_dsts(asus_armoury.gpu_mux_dev_id, &result); + if (err) { + pr_warn("Failed to get GPU MUX status: %d\n", result); + return result; + } + if (!result && enable) { + pr_warn("Can not enable eGPU when the MUX is in dGPU mode\n"); + return -ENODEV; + } + } + + err = armoury_wmi_set_devstate(attr, enable, ASUS_WMI_DEVID_EGPU); + if (err) + return err; + + sysfs_notify(kobj, NULL, attr->attr.name); + + return count; +} +WMI_SHOW_INT(egpu_enable_current_value, "%d\n", ASUS_WMI_DEVID_EGPU); +ATTR_GROUP_BOOL_CUSTOM(egpu_enable, "egpu_enable", "Enable the eGPU (also disables dGPU)"); + +/* Device memory available to APU */ + +static ssize_t apu_mem_current_value_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + int err; + u32 mem; + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_APU_MEM, &mem); + if (err) + return err; + + switch (mem) { + case 0x100: + mem = 0; + break; + case 0x102: + mem = 1; + break; + case 0x103: + mem = 2; + break; + case 0x104: + mem = 3; + break; + case 0x105: + mem = 4; + break; + case 0x106: + /* This is out of order and looks wrong but is correct */ + mem = 8; + break; + case 0x107: + mem = 5; + break; + case 0x108: + mem = 6; + break; + case 0x109: + mem = 7; + break; + default: + mem = 4; + break; + } + + return sysfs_emit(buf, "%u\n", mem); +} + +static ssize_t apu_mem_current_value_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int result, err; + u32 requested, mem; + + result = kstrtou32(buf, 10, &requested); + if (result) + return result; + + switch (requested) { + case 0: + mem = 0x000; + break; + case 1: + mem = 0x102; + break; + case 2: + mem = 0x103; + break; + case 3: + mem = 0x104; + break; + case 4: + mem = 0x105; + break; + case 5: + mem = 0x107; + break; + case 6: + mem = 0x108; + break; + case 7: + mem = 0x109; + break; + case 8: + /* This is out of order and looks wrong but is correct */ + mem = 0x106; + break; + default: + return -EIO; + } + + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_APU_MEM, mem, &result); + if (err) { + pr_warn("Failed to set apu_mem: %d\n", err); + return err; + } + + pr_info("APU memory changed to %uGB, reboot required\n", requested); + sysfs_notify(kobj, NULL, attr->attr.name); + + asus_set_reboot_and_signal_event(); + + return count; +} + +static ssize_t apu_mem_possible_values_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "0;1;2;3;4;5;6;7;8\n"); +} +ATTR_GROUP_ENUM_CUSTOM(apu_mem, "apu_mem", "Set available system RAM (in GB) for the APU to use"); + +static int init_max_cpu_cores(void) +{ + u32 cores; + int err; + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_CORES_MAX, &cores); + if (err) + return err; + + cores &= ~ASUS_WMI_DSTS_PRESENCE_BIT; + asus_armoury.cpu_cores->max_power_cores = FIELD_GET(ASUS_POWER_CORE_MASK, cores); + asus_armoury.cpu_cores->max_perf_cores = FIELD_GET(ASUS_PERF_CORE_MASK, cores); + + err = asus_wmi_get_devstate_dsts(ASUS_WMI_DEVID_CORES, &cores); + if (err) { + pr_err("Could not get CPU core count: error %d", err); + return err; + } + + asus_armoury.cpu_cores->cur_perf_cores = FIELD_GET(ASUS_PERF_CORE_MASK, cores); + asus_armoury.cpu_cores->cur_power_cores = FIELD_GET(ASUS_POWER_CORE_MASK, cores); + + asus_armoury.cpu_cores->min_perf_cores = CPU_PERF_CORE_COUNT_MIN; + asus_armoury.cpu_cores->min_power_cores = CPU_POWR_CORE_COUNT_MIN; + + return 0; +} + +static ssize_t cores_value_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf, + enum cpu_core_type core_type, enum cpu_core_value core_value) +{ + u32 cores; + + switch (core_value) { + case CPU_CORE_DEFAULT: + case CPU_CORE_MAX: + if (core_type == CPU_CORE_PERF) + return sysfs_emit(buf, "%d\n", + asus_armoury.cpu_cores->max_perf_cores); + else + return sysfs_emit(buf, "%d\n", + asus_armoury.cpu_cores->max_power_cores); + case CPU_CORE_MIN: + if (core_type == CPU_CORE_PERF) + return sysfs_emit(buf, "%d\n", + asus_armoury.cpu_cores->min_perf_cores); + else + return sysfs_emit(buf, "%d\n", + asus_armoury.cpu_cores->min_power_cores); + default: + break; + } + + if (core_type == CPU_CORE_PERF) + cores = asus_armoury.cpu_cores->cur_perf_cores; + else + cores = asus_armoury.cpu_cores->cur_power_cores; + + return sysfs_emit(buf, "%d\n", cores); +} + +static ssize_t cores_current_value_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, enum cpu_core_type core_type) +{ + u32 new_cores, perf_cores, power_cores, out_val, min, max; + int result, err; + + result = kstrtou32(buf, 10, &new_cores); + if (result) + return result; + + mutex_lock(&asus_armoury.cpu_core_mutex); + + if (core_type == CPU_CORE_PERF) { + perf_cores = new_cores; + power_cores = out_val = asus_armoury.cpu_cores->cur_power_cores; + min = asus_armoury.cpu_cores->min_perf_cores; + max = asus_armoury.cpu_cores->max_perf_cores; + } else { + perf_cores = asus_armoury.cpu_cores->cur_perf_cores; + power_cores = out_val = new_cores; + min = asus_armoury.cpu_cores->min_power_cores; + max = asus_armoury.cpu_cores->max_power_cores; + } + + if (new_cores < min || new_cores > max) { + mutex_unlock(&asus_armoury.cpu_core_mutex); + return -EINVAL; + } + + out_val = 0; + out_val |= FIELD_PREP(ASUS_PERF_CORE_MASK, perf_cores); + out_val |= FIELD_PREP(ASUS_POWER_CORE_MASK, power_cores); + + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_CORES, out_val, &result); + + if (err) { + pr_warn("Failed to set CPU core count: %d\n", err); + mutex_unlock(&asus_armoury.cpu_core_mutex); + return err; + } + + if (result > 1) { + pr_warn("Failed to set CPU core count (result): 0x%x\n", result); + mutex_unlock(&asus_armoury.cpu_core_mutex); + return -EIO; + } + + pr_info("CPU core count changed, reboot required\n"); + mutex_unlock(&asus_armoury.cpu_core_mutex); + + sysfs_notify(kobj, NULL, attr->attr.name); + asus_set_reboot_and_signal_event(); + + return 0; +} + +static ssize_t cores_performance_min_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_MIN); +} + +static ssize_t cores_performance_max_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_MAX); +} + +static ssize_t cores_performance_default_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_DEFAULT); +} + +static ssize_t cores_performance_current_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_PERF, CPU_CORE_CURRENT); +} + +static ssize_t cores_performance_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int err; + + err = cores_current_value_store(kobj, attr, buf, CPU_CORE_PERF); + if (err) + return err; + + return count; +} +ATTR_GROUP_CORES_RW(cores_performance, "cores_performance", + "Set the max available performance cores"); + +static ssize_t cores_efficiency_min_value_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_MIN); +} + +static ssize_t cores_efficiency_max_value_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_MAX); +} + +static ssize_t cores_efficiency_default_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_DEFAULT); +} + +static ssize_t cores_efficiency_current_value_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return cores_value_show(kobj, attr, buf, CPU_CORE_POWER, CPU_CORE_CURRENT); +} + +static ssize_t cores_efficiency_current_value_store(struct kobject *kobj, + struct kobj_attribute *attr, const char *buf, + size_t count) +{ + int err; + + err = cores_current_value_store(kobj, attr, buf, CPU_CORE_POWER); + if (err) + return err; + + return count; +} +ATTR_GROUP_CORES_RW(cores_efficiency, "cores_efficiency", + "Set the max available efficiency cores"); + +/* Define helper to access the current power mode tunable values */ +static inline struct rog_tunables *get_current_tunables(void) +{ + return asus_armoury + .rog_tunables[power_supply_is_system_supplied() ? 1 : 0]; +} + +/* Simple attribute creation */ +ATTR_GROUP_ROG_TUNABLE(ppt_pl1_spl, ATTR_PPT_PL1_SPL, ASUS_WMI_DEVID_PPT_PL1_SPL, + "Set the CPU slow package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_pl2_sppt, ATTR_PPT_PL2_SPPT, ASUS_WMI_DEVID_PPT_PL2_SPPT, + "Set the CPU fast package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_pl3_fppt, ATTR_PPT_PL3_FPPT, ASUS_WMI_DEVID_PPT_FPPT, + "Set the CPU fastest package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_apu_sppt, ATTR_PPT_APU_SPPT, ASUS_WMI_DEVID_PPT_APU_SPPT, + "Set the APU package limit"); +ATTR_GROUP_ROG_TUNABLE(ppt_platform_sppt, ATTR_PPT_PLATFORM_SPPT, ASUS_WMI_DEVID_PPT_PLAT_SPPT, + "Set the platform package limit"); +ATTR_GROUP_ROG_TUNABLE(nv_dynamic_boost, ATTR_NV_DYNAMIC_BOOST, ASUS_WMI_DEVID_NV_DYN_BOOST, + "Set the Nvidia dynamic boost limit"); +ATTR_GROUP_ROG_TUNABLE(nv_temp_target, ATTR_NV_TEMP_TARGET, ASUS_WMI_DEVID_NV_THERM_TARGET, + "Set the Nvidia max thermal limit"); +ATTR_GROUP_ROG_TUNABLE(nv_tgp, "nv_tgp", ASUS_WMI_DEVID_DGPU_SET_TGP, + "Set the additional TGP on top of the base TGP"); +ATTR_GROUP_INT_VALUE_ONLY_RO(nv_base_tgp, ATTR_NV_BASE_TGP, ASUS_WMI_DEVID_DGPU_BASE_TGP, + "Read the base TGP value"); + + +ATTR_GROUP_ENUM_INT_RO(charge_mode, "charge_mode", ASUS_WMI_DEVID_CHARGE_MODE, "0;1;2", + "Show the current mode of charging"); + +ATTR_GROUP_BOOL_RW(boot_sound, "boot_sound", ASUS_WMI_DEVID_BOOT_SOUND, + "Set the boot POST sound"); +ATTR_GROUP_BOOL_RW(mcu_powersave, "mcu_powersave", ASUS_WMI_DEVID_MCU_POWERSAVE, + "Set MCU powersaving mode"); +ATTR_GROUP_BOOL_RW(panel_od, "panel_overdrive", ASUS_WMI_DEVID_PANEL_OD, + "Set the panel refresh overdrive"); +ATTR_GROUP_BOOL_RW(panel_hd_mode, "panel_hd_mode", ASUS_WMI_DEVID_PANEL_HD, + "Set the panel HD mode to UHD<0> or FHD<1>"); +ATTR_GROUP_BOOL_RW(screen_auto_brightness, "screen_auto_brightness", + ASUS_WMI_DEVID_SCREEN_AUTO_BRIGHTNESS, + "Set the panel brightness to Off<0> or On<1>"); +ATTR_GROUP_BOOL_RO(egpu_connected, "egpu_connected", ASUS_WMI_DEVID_EGPU_CONNECTED, + "Show the eGPU connection status"); + +/* If an attribute does not require any special case handling add it here */ +static const struct asus_attr_group armoury_attr_groups[] = { + { &egpu_connected_attr_group, ASUS_WMI_DEVID_EGPU_CONNECTED }, + { &egpu_enable_attr_group, ASUS_WMI_DEVID_EGPU }, + { &dgpu_disable_attr_group, ASUS_WMI_DEVID_DGPU }, + { &apu_mem_attr_group, ASUS_WMI_DEVID_APU_MEM }, + { &cores_efficiency_attr_group, ASUS_WMI_DEVID_CORES_MAX }, + { &cores_performance_attr_group, ASUS_WMI_DEVID_CORES_MAX }, + + { &ppt_pl1_spl_attr_group, ASUS_WMI_DEVID_PPT_PL1_SPL }, + { &ppt_pl2_sppt_attr_group, ASUS_WMI_DEVID_PPT_PL2_SPPT }, + { &ppt_pl3_fppt_attr_group, ASUS_WMI_DEVID_PPT_FPPT }, + { &ppt_apu_sppt_attr_group, ASUS_WMI_DEVID_PPT_APU_SPPT }, + { &ppt_platform_sppt_attr_group, ASUS_WMI_DEVID_PPT_PLAT_SPPT }, + { &nv_dynamic_boost_attr_group, ASUS_WMI_DEVID_NV_DYN_BOOST }, + { &nv_temp_target_attr_group, ASUS_WMI_DEVID_NV_THERM_TARGET }, + { &nv_base_tgp_attr_group, ASUS_WMI_DEVID_DGPU_BASE_TGP }, + { &nv_tgp_attr_group, ASUS_WMI_DEVID_DGPU_SET_TGP }, + + { &charge_mode_attr_group, ASUS_WMI_DEVID_CHARGE_MODE }, + { &boot_sound_attr_group, ASUS_WMI_DEVID_BOOT_SOUND }, + { &mcu_powersave_attr_group, ASUS_WMI_DEVID_MCU_POWERSAVE }, + { &panel_od_attr_group, ASUS_WMI_DEVID_PANEL_OD }, + { &panel_hd_mode_attr_group, ASUS_WMI_DEVID_PANEL_HD }, +}; + +/** + * is_power_tunable_attr - Determines if an attribute is a power-related tunable + * @name: The name of the attribute to check + * + * This function checks if the given attribute name is related to power tuning. + * + * Return: true if the attribute is a power-related tunable, false otherwise + */ +static bool is_power_tunable_attr(const char *name) +{ + static const char * const power_tunable_attrs[] = { + ATTR_PPT_PL1_SPL, ATTR_PPT_PL2_SPPT, + ATTR_PPT_PL3_FPPT, ATTR_PPT_APU_SPPT, + ATTR_PPT_PLATFORM_SPPT, ATTR_NV_DYNAMIC_BOOST, + ATTR_NV_TEMP_TARGET, ATTR_NV_BASE_TGP, + ATTR_NV_TGP + }; + + for (int i = 0; i < ARRAY_SIZE(power_tunable_attrs); i++) { + if (!strcmp(name, power_tunable_attrs[i])) + return true; + } + + return false; +} + +/** + * has_valid_limit - Checks if a power-related attribute has a valid limit value + * @name: The name of the attribute to check + * @limits: Pointer to the power_limits structure containing limit values + * + * This function checks if a power-related attribute has a valid limit value. + * It returns false if limits is NULL or if the corresponding limit value is zero. + * + * Return: true if the attribute has a valid limit value, false otherwise + */ +static bool has_valid_limit(const char *name, const struct power_limits *limits) +{ + u32 limit_value = 0; + + if (!limits) + return false; + + if (!strcmp(name, ATTR_PPT_PL1_SPL)) + limit_value = limits->ppt_pl1_spl_max; + else if (!strcmp(name, ATTR_PPT_PL2_SPPT)) + limit_value = limits->ppt_pl2_sppt_max; + else if (!strcmp(name, ATTR_PPT_PL3_FPPT)) + limit_value = limits->ppt_pl3_fppt_max; + else if (!strcmp(name, ATTR_PPT_APU_SPPT)) + limit_value = limits->ppt_apu_sppt_max; + else if (!strcmp(name, ATTR_PPT_PLATFORM_SPPT)) + limit_value = limits->ppt_platform_sppt_max; + else if (!strcmp(name, ATTR_NV_DYNAMIC_BOOST)) + limit_value = limits->nv_dynamic_boost_max; + else if (!strcmp(name, ATTR_NV_TEMP_TARGET)) + limit_value = limits->nv_temp_target_max; + else if (!strcmp(name, ATTR_NV_BASE_TGP) || + !strcmp(name, ATTR_NV_TGP)) + limit_value = limits->nv_tgp_max; + + return limit_value > 0; +} + +static int asus_fw_attr_add(void) +{ + const struct power_limits *limits; + bool should_create; + const char *name; + int err, i; + + asus_armoury.fw_attr_dev = device_create(&firmware_attributes_class, NULL, MKDEV(0, 0), + NULL, "%s", DRIVER_NAME); + if (IS_ERR(asus_armoury.fw_attr_dev)) { + err = PTR_ERR(asus_armoury.fw_attr_dev); + goto fail_class_get; + } + + asus_armoury.fw_attr_kset = kset_create_and_add("attributes", NULL, + &asus_armoury.fw_attr_dev->kobj); + if (!asus_armoury.fw_attr_kset) { + err = -ENOMEM; + goto err_destroy_classdev; + } + + err = sysfs_create_file(&asus_armoury.fw_attr_kset->kobj, &pending_reboot.attr); + if (err) { + pr_err("Failed to create sysfs level attributes\n"); + goto err_destroy_kset; + } + + asus_armoury.mini_led_dev_id = 0; + if (asus_wmi_is_present(ASUS_WMI_DEVID_MINI_LED_MODE)) + asus_armoury.mini_led_dev_id = ASUS_WMI_DEVID_MINI_LED_MODE; + else if (asus_wmi_is_present(ASUS_WMI_DEVID_MINI_LED_MODE2)) + asus_armoury.mini_led_dev_id = ASUS_WMI_DEVID_MINI_LED_MODE2; + + if (asus_armoury.mini_led_dev_id) { + err = sysfs_create_group(&asus_armoury.fw_attr_kset->kobj, + &mini_led_mode_attr_group); + if (err) { + pr_err("Failed to create sysfs-group for mini_led\n"); + goto err_remove_file; + } + } + + asus_armoury.gpu_mux_dev_id = 0; + if (asus_wmi_is_present(ASUS_WMI_DEVID_GPU_MUX)) + asus_armoury.gpu_mux_dev_id = ASUS_WMI_DEVID_GPU_MUX; + else if (asus_wmi_is_present(ASUS_WMI_DEVID_GPU_MUX_VIVO)) + asus_armoury.gpu_mux_dev_id = ASUS_WMI_DEVID_GPU_MUX_VIVO; + + if (asus_armoury.gpu_mux_dev_id) { + err = sysfs_create_group(&asus_armoury.fw_attr_kset->kobj, + &gpu_mux_mode_attr_group); + if (err) { + pr_err("Failed to create sysfs-group for gpu_mux\n"); + goto err_remove_mini_led_group; + } + } + + for (i = 0; i < ARRAY_SIZE(armoury_attr_groups); i++) { + if (!asus_wmi_is_present(armoury_attr_groups[i].wmi_devid)) + continue; + + /* Always create by default, unless PPT is not present */ + should_create = true; + name = armoury_attr_groups[i].attr_group->name; + + /* Check if this is a power-related tunable requiring limits */ + if (asus_armoury.rog_tunables[1] && asus_armoury.rog_tunables[1]->power_limits && + is_power_tunable_attr(name)) { + limits = asus_armoury.rog_tunables[1]->power_limits; + /* Check only AC, if DC is not present then AC won't be either */ + should_create = has_valid_limit(name, limits); + if (!should_create) { + pr_debug( + "Missing max value on %s for tunable: %s\n", + dmi_get_system_info(DMI_BOARD_NAME), + name); + } + } + + if (should_create) { + err = sysfs_create_group( + &asus_armoury.fw_attr_kset->kobj, + armoury_attr_groups[i].attr_group); + if (err) { + pr_err("Failed to create sysfs-group for %s\n", + armoury_attr_groups[i].attr_group->name); + goto err_remove_groups; + } + } + } + + return 0; + +err_remove_groups: + while (--i >= 0) { + if (asus_wmi_is_present(armoury_attr_groups[i].wmi_devid)) + sysfs_remove_group(&asus_armoury.fw_attr_kset->kobj, + armoury_attr_groups[i].attr_group); + } + if (asus_armoury.gpu_mux_dev_id) + sysfs_remove_group(&asus_armoury.fw_attr_kset->kobj, &gpu_mux_mode_attr_group); +err_remove_mini_led_group: + if (asus_armoury.mini_led_dev_id) + sysfs_remove_group(&asus_armoury.fw_attr_kset->kobj, &mini_led_mode_attr_group); +err_remove_file: + sysfs_remove_file(&asus_armoury.fw_attr_kset->kobj, &pending_reboot.attr); +err_destroy_kset: + kset_unregister(asus_armoury.fw_attr_kset); +err_destroy_classdev: +fail_class_get: + device_destroy(&firmware_attributes_class, MKDEV(0, 0)); + return err; +} + +/* Init / exit ****************************************************************/ + +/* Set up the min/max and defaults for ROG tunables */ +static void init_rog_tunables(void) +{ + const struct power_limits *ac_limits, *dc_limits; + const struct power_data *power_data; + const struct dmi_system_id *dmi_id; + bool ac_initialized = false, dc_initialized = false; + + /* Match the system against the power_limits table */ + dmi_id = dmi_first_match(power_limits); + if (!dmi_id) { + pr_warn("No matching power limits found for this system\n"); + return; + } + + /* Get the power data for this system */ + power_data = dmi_id->driver_data; + if (!power_data) { + pr_info("No power data available for this system\n"); + return; + } + + /* Initialize AC power tunables */ + ac_limits = power_data->ac_data; + if (ac_limits) { + asus_armoury.rog_tunables[1] = + kzalloc(sizeof(struct rog_tunables), GFP_KERNEL); + if (!asus_armoury.rog_tunables[1]) + goto err_nomem; + + asus_armoury.rog_tunables[1]->power_limits = ac_limits; + + /* Set initial AC values */ + asus_armoury.rog_tunables[1]->ppt_pl1_spl = + ac_limits->ppt_pl1_spl_def ? + ac_limits->ppt_pl1_spl_def : + ac_limits->ppt_pl1_spl_max; + + asus_armoury.rog_tunables[1]->ppt_pl2_sppt = + ac_limits->ppt_pl2_sppt_def ? + ac_limits->ppt_pl2_sppt_def : + ac_limits->ppt_pl2_sppt_max; + + asus_armoury.rog_tunables[1]->ppt_pl3_fppt = + ac_limits->ppt_pl3_fppt_def ? + ac_limits->ppt_pl3_fppt_def : + ac_limits->ppt_pl3_fppt_max; + + asus_armoury.rog_tunables[1]->ppt_apu_sppt = + ac_limits->ppt_apu_sppt_def ? + ac_limits->ppt_apu_sppt_def : + ac_limits->ppt_apu_sppt_max; + + asus_armoury.rog_tunables[1]->ppt_platform_sppt = + ac_limits->ppt_platform_sppt_def ? + ac_limits->ppt_platform_sppt_def : + ac_limits->ppt_platform_sppt_max; + + asus_armoury.rog_tunables[1]->nv_dynamic_boost = + ac_limits->nv_dynamic_boost_max; + asus_armoury.rog_tunables[1]->nv_temp_target = + ac_limits->nv_temp_target_max; + asus_armoury.rog_tunables[1]->nv_tgp = ac_limits->nv_tgp_max; + + ac_initialized = true; + pr_debug("AC power limits initialized for %s\n", dmi_id->matches[0].substr); + } + + /* Initialize DC power tunables */ + dc_limits = power_data->dc_data; + if (dc_limits) { + asus_armoury.rog_tunables[0] = + kzalloc(sizeof(struct rog_tunables), GFP_KERNEL); + if (!asus_armoury.rog_tunables[0]) { + if (ac_initialized) + kfree(asus_armoury.rog_tunables[1]); + goto err_nomem; + } + + asus_armoury.rog_tunables[0]->power_limits = dc_limits; + + /* Set initial DC values */ + asus_armoury.rog_tunables[0]->ppt_pl1_spl = + dc_limits->ppt_pl1_spl_def ? + dc_limits->ppt_pl1_spl_def : + dc_limits->ppt_pl1_spl_max; + + asus_armoury.rog_tunables[0]->ppt_pl2_sppt = + dc_limits->ppt_pl2_sppt_def ? + dc_limits->ppt_pl2_sppt_def : + dc_limits->ppt_pl2_sppt_max; + + asus_armoury.rog_tunables[0]->ppt_pl3_fppt = + dc_limits->ppt_pl3_fppt_def ? + dc_limits->ppt_pl3_fppt_def : + dc_limits->ppt_pl3_fppt_max; + + asus_armoury.rog_tunables[0]->ppt_apu_sppt = + dc_limits->ppt_apu_sppt_def ? + dc_limits->ppt_apu_sppt_def : + dc_limits->ppt_apu_sppt_max; + + asus_armoury.rog_tunables[0]->ppt_platform_sppt = + dc_limits->ppt_platform_sppt_def ? + dc_limits->ppt_platform_sppt_def : + dc_limits->ppt_platform_sppt_max; + + asus_armoury.rog_tunables[0]->nv_dynamic_boost = + dc_limits->nv_dynamic_boost_max; + asus_armoury.rog_tunables[0]->nv_temp_target = + dc_limits->nv_temp_target_max; + asus_armoury.rog_tunables[0]->nv_tgp = dc_limits->nv_tgp_max; + + dc_initialized = true; + pr_debug("DC power limits initialized for %s\n", dmi_id->matches[0].substr); + } + + if (!ac_initialized) + pr_debug("No AC PPT limits defined\n"); + + if (!dc_initialized) + pr_debug("No DC PPT limits defined\n"); + + return; + +err_nomem: + pr_err("Failed to allocate memory for tunables\n"); +} + +static int __init asus_fw_init(void) +{ + char *wmi_uid; + int err; + + wmi_uid = wmi_get_acpi_device_uid(ASUS_WMI_MGMT_GUID); + if (!wmi_uid) + return -ENODEV; + + /* + * if equal to "ASUSWMI" then it's DCTS that can't be used for this + * driver, DSTS is required. + */ + if (!strcmp(wmi_uid, ASUS_ACPI_UID_ASUSWMI)) + return -ENODEV; + + if (asus_wmi_is_present(ASUS_WMI_DEVID_CORES_MAX)) { + asus_armoury.cpu_cores = kzalloc(sizeof(struct cpu_cores), GFP_KERNEL); + if (!asus_armoury.cpu_cores) + return -ENOMEM; + + err = init_max_cpu_cores(); + if (err) { + kfree(asus_armoury.cpu_cores); + pr_err("Could not initialise CPU core control %d\n", err); + return err; + } + } + + init_rog_tunables(); + + /* Must always be last step to ensure data is available */ + return asus_fw_attr_add(); +} + +static void __exit asus_fw_exit(void) +{ + sysfs_remove_file(&asus_armoury.fw_attr_kset->kobj, &pending_reboot.attr); + kset_unregister(asus_armoury.fw_attr_kset); + device_destroy(&firmware_attributes_class, MKDEV(0, 0)); + + kfree(asus_armoury.rog_tunables[0]); + kfree(asus_armoury.rog_tunables[1]); +} + +module_init(asus_fw_init); +module_exit(asus_fw_exit); + +MODULE_IMPORT_NS("ASUS_WMI"); +MODULE_AUTHOR("Luke Jones "); +MODULE_DESCRIPTION("ASUS BIOS Configuration Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("wmi:" ASUS_NB_WMI_EVENT_GUID); diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h new file mode 100644 index 000000000000..438768ea14cc --- /dev/null +++ b/drivers/platform/x86/asus-armoury.h @@ -0,0 +1,1278 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Definitions for kernel modules using asus-armoury driver + * + * Copyright (c) 2024 Luke Jones + */ + +#ifndef _ASUS_ARMOURY_H_ +#define _ASUS_ARMOURY_H_ + +#include +#include +#include + +#define DRIVER_NAME "asus-armoury" + +#define __ASUS_ATTR_RO(_func, _name) \ + { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = _func##_##_name##_show, \ + } + +#define __ASUS_ATTR_RO_AS(_name, _show) \ + { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = _show, \ + } + +#define __ASUS_ATTR_RW(_func, _name) \ + __ATTR(_name, 0644, _func##_##_name##_show, _func##_##_name##_store) + +#define __WMI_STORE_INT(_attr, _min, _max, _wmi) \ + static ssize_t _attr##_store(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + return attr_uint_store(kobj, attr, buf, count, _min, \ + _max, NULL, _wmi); \ + } + +#define WMI_SHOW_INT(_attr, _fmt, _wmi) \ + static ssize_t _attr##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ + { \ + u32 result; \ + int err; \ + \ + err = asus_wmi_get_devstate_dsts(_wmi, &result); \ + if (err) \ + return err; \ + return sysfs_emit(buf, _fmt, \ + result & ~ASUS_WMI_DSTS_PRESENCE_BIT); \ + } + +/* Create functions and attributes for use in other macros or on their own */ + +/* Shows a formatted static variable */ +#define __ATTR_SHOW_FMT(_prop, _attrname, _fmt, _val) \ + static ssize_t _attrname##_##_prop##_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + return sysfs_emit(buf, _fmt, _val); \ + } \ + static struct kobj_attribute attr_##_attrname##_##_prop = \ + __ASUS_ATTR_RO(_attrname, _prop) + +#define __ATTR_RO_INT_GROUP_ENUM(_attrname, _wmi, _fsname, _possible, _dispname)\ + WMI_SHOW_INT(_attrname##_current_value, "%d\n", _wmi); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RO(_attrname, current_value); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + __ATTR_SHOW_FMT(possible_values, _attrname, "%s\n", _possible); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +#define __ATTR_RW_INT_GROUP_ENUM(_attrname, _minv, _maxv, _wmi, _fsname,\ + _possible, _dispname) \ + __WMI_STORE_INT(_attrname##_current_value, _minv, _maxv, _wmi); \ + WMI_SHOW_INT(_attrname##_current_value, "%d\n", _wmi); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + __ATTR_SHOW_FMT(possible_values, _attrname, "%s\n", _possible); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* Boolean style enumeration, base macro. Requires adding show/store */ +#define __ATTR_GROUP_ENUM(_attrname, _fsname, _possible, _dispname) \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + __ATTR_SHOW_FMT(possible_values, _attrname, "%s\n", _possible); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +#define ATTR_GROUP_BOOL_RO(_attrname, _fsname, _wmi, _dispname) \ + __ATTR_RO_INT_GROUP_ENUM(_attrname, _wmi, _fsname, "0;1", _dispname) + + +#define ATTR_GROUP_BOOL_RW(_attrname, _fsname, _wmi, _dispname) \ + __ATTR_RW_INT_GROUP_ENUM(_attrname, 0, 1, _wmi, _fsname, "0;1", _dispname) + +#define ATTR_GROUP_ENUM_INT_RO(_attrname, _fsname, _wmi, _possible, _dispname) \ + __ATTR_RO_INT_GROUP_ENUM(_attrname, _wmi, _fsname, _possible, _dispname) + +/* + * Requires _current_value_show(), _current_value_show() + */ +#define ATTR_GROUP_BOOL_CUSTOM(_attrname, _fsname, _dispname) \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + __ATTR_GROUP_ENUM(_attrname, _fsname, "0;1", _dispname) + +/* + * Requires _current_value_show(), _current_value_show() + * and _possible_values_show() + */ +#define ATTR_GROUP_ENUM_CUSTOM(_attrname, _fsname, _dispname) \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + static struct kobj_attribute attr_##_attrname##_possible_values = \ + __ASUS_ATTR_RO(_attrname, possible_values); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, enum_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_possible_values.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* CPU core attributes need a little different in setup */ +#define ATTR_GROUP_CORES_RW(_attrname, _fsname, _dispname) \ + __ATTR_SHOW_FMT(scalar_increment, _attrname, "%d\n", 1); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RW(_attrname, current_value); \ + static struct kobj_attribute attr_##_attrname##_default_value = \ + __ASUS_ATTR_RO(_attrname, default_value); \ + static struct kobj_attribute attr_##_attrname##_min_value = \ + __ASUS_ATTR_RO(_attrname, min_value); \ + static struct kobj_attribute attr_##_attrname##_max_value = \ + __ASUS_ATTR_RO(_attrname, max_value); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, int_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_default_value.attr, \ + &attr_##_attrname##_min_value.attr, \ + &attr_##_attrname##_max_value.attr, \ + &attr_##_attrname##_scalar_increment.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +#define ATTR_GROUP_INT_VALUE_ONLY_RO(_attrname, _fsname, _wmi, _dispname) \ + WMI_SHOW_INT(_attrname##_current_value, "%d\n", _wmi); \ + static struct kobj_attribute attr_##_attrname##_current_value = \ + __ASUS_ATTR_RO(_attrname, current_value); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, int_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_type.attr, NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* + * ROG PPT attributes need a little different in setup as they + * require rog_tunables members. + */ + +#define __ROG_TUNABLE_SHOW(_prop, _attrname, _val) \ + static ssize_t _attrname##_##_prop##_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables || !tunables->power_limits) \ + return -ENODEV; \ + \ + return sysfs_emit(buf, "%d\n", tunables->power_limits->_val); \ + } \ + static struct kobj_attribute attr_##_attrname##_##_prop = \ + __ASUS_ATTR_RO(_attrname, _prop) + +#define __ROG_TUNABLE_SHOW_DEFAULT(_attrname) \ + static ssize_t _attrname##_default_value_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables || !tunables->power_limits) \ + return -ENODEV; \ + \ + return sysfs_emit( \ + buf, "%d\n", \ + tunables->power_limits->_attrname##_def ? \ + tunables->power_limits->_attrname##_def : \ + tunables->power_limits->_attrname##_max); \ + } \ + static struct kobj_attribute attr_##_attrname##_default_value = \ + __ASUS_ATTR_RO(_attrname, default_value) + +#define __ROG_TUNABLE_RW(_attr, _wmi) \ + static ssize_t _attr##_current_value_store( \ + struct kobject *kobj, struct kobj_attribute *attr, \ + const char *buf, size_t count) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables || !tunables->power_limits) \ + return -ENODEV; \ + \ + return attr_uint_store(kobj, attr, buf, count, \ + tunables->power_limits->_attr##_min, \ + tunables->power_limits->_attr##_max, \ + &tunables->_attr, _wmi); \ + } \ + static ssize_t _attr##_current_value_show( \ + struct kobject *kobj, struct kobj_attribute *attr, char *buf) \ + { \ + struct rog_tunables *tunables = get_current_tunables(); \ + \ + if (!tunables) \ + return -ENODEV; \ + \ + return sysfs_emit(buf, "%u\n", tunables->_attr); \ + } \ + static struct kobj_attribute attr_##_attr##_current_value = \ + __ASUS_ATTR_RW(_attr, current_value) + +#define ATTR_GROUP_ROG_TUNABLE(_attrname, _fsname, _wmi, _dispname) \ + __ROG_TUNABLE_RW(_attrname, _wmi); \ + __ROG_TUNABLE_SHOW_DEFAULT(_attrname); \ + __ROG_TUNABLE_SHOW(min_value, _attrname, _attrname##_min); \ + __ROG_TUNABLE_SHOW(max_value, _attrname, _attrname##_max); \ + __ATTR_SHOW_FMT(scalar_increment, _attrname, "%d\n", 1); \ + __ATTR_SHOW_FMT(display_name, _attrname, "%s\n", _dispname); \ + static struct kobj_attribute attr_##_attrname##_type = \ + __ASUS_ATTR_RO_AS(type, int_type_show); \ + static struct attribute *_attrname##_attrs[] = { \ + &attr_##_attrname##_current_value.attr, \ + &attr_##_attrname##_default_value.attr, \ + &attr_##_attrname##_min_value.attr, \ + &attr_##_attrname##_max_value.attr, \ + &attr_##_attrname##_scalar_increment.attr, \ + &attr_##_attrname##_display_name.attr, \ + &attr_##_attrname##_type.attr, \ + NULL \ + }; \ + static const struct attribute_group _attrname##_attr_group = { \ + .name = _fsname, .attrs = _attrname##_attrs \ + } + +/* Default is always the maximum value unless *_def is specified */ +struct power_limits { + u8 ppt_pl1_spl_min; + u8 ppt_pl1_spl_def; + u8 ppt_pl1_spl_max; + u8 ppt_pl2_sppt_min; + u8 ppt_pl2_sppt_def; + u8 ppt_pl2_sppt_max; + u8 ppt_pl3_fppt_min; + u8 ppt_pl3_fppt_def; + u8 ppt_pl3_fppt_max; + u8 ppt_apu_sppt_min; + u8 ppt_apu_sppt_def; + u8 ppt_apu_sppt_max; + u8 ppt_platform_sppt_min; + u8 ppt_platform_sppt_def; + u8 ppt_platform_sppt_max; + /* Nvidia GPU specific, default is always max */ + u8 nv_dynamic_boost_def; // unused. exists for macro + u8 nv_dynamic_boost_min; + u8 nv_dynamic_boost_max; + u8 nv_temp_target_def; // unused. exists for macro + u8 nv_temp_target_min; + u8 nv_temp_target_max; + u8 nv_tgp_def; // unused. exists for macro + u8 nv_tgp_min; + u8 nv_tgp_max; +}; + +struct power_data { + const struct power_limits *ac_data; + const struct power_limits *dc_data; + bool requires_fan_curve; +}; + +/* + * For each avilable attribute there must be a min and a max. + * _def is not required and will be assumed to be default == max if missing. + */ +static const struct dmi_system_id power_limits[] = { + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA401W"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 75, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 30, + .ppt_pl2_sppt_min = 31, + .ppt_pl2_sppt_max = 44, + .ppt_pl3_fppt_min = 45, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507N"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 45, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80 + }, + .dc_data = NULL + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507X"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 45, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA507Z"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 105, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 15, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 60, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA607P"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 30, + .ppt_pl1_spl_def = 100, + .ppt_pl1_spl_max = 135, + .ppt_pl2_sppt_min = 30, + .ppt_pl2_sppt_def = 115, + .ppt_pl2_sppt_max = 135, + .ppt_pl3_fppt_min = 30, + .ppt_pl3_fppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 115, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_def = 45, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_def = 60, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 25, + .ppt_pl3_fppt_max = 80, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA617NS"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 120 + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 25, + .ppt_apu_sppt_max = 35, + .ppt_platform_sppt_min = 45, + .ppt_platform_sppt_max = 100 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA617NT"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 115 + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 45, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 50 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FA617XS"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 120, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 25, + .ppt_apu_sppt_max = 35, + .ppt_platform_sppt_min = 45, + .ppt_platform_sppt_max = 100, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "FX507Z"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 15, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 60, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA401Q"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 80, + }, + .dc_data = NULL + }, + }, + { + .matches = { + // This model is full AMD. No Nvidia dGPU. + DMI_MATCH(DMI_BOARD_NAME, "GA402R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_apu_sppt_min = 15, + .ppt_apu_sppt_max = 80, + .ppt_platform_sppt_min = 30, + .ppt_platform_sppt_max = 115, + }, + .dc_data = &(struct power_limits) { + .ppt_apu_sppt_min = 25, + .ppt_apu_sppt_def = 30, + .ppt_apu_sppt_max = 45, + .ppt_platform_sppt_min = 40, + .ppt_platform_sppt_max = 60, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA402X"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 35, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_def = 65, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 35, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA403U"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 65, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 35, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA503R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 35, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 65, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 25, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 60, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GA605W"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 31, + .ppt_pl2_sppt_max = 44, + .ppt_pl3_fppt_min = 45, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GU603Z"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 60, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 135, + /* Only allowed in AC mode */ + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 40, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 40, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GU604V"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 65, + .ppt_pl1_spl_max = 120, + .ppt_pl2_sppt_min = 65, + .ppt_pl2_sppt_max = 150, + /* Only allowed in AC mode */ + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 40, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 40, + .ppt_pl2_sppt_max = 60, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GU605M"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 38, + .ppt_pl2_sppt_max = 53, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV301Q"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 65, + .ppt_pl2_sppt_max = 80, + }, + .dc_data = NULL + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV301R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 45, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 54, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 35, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV601R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 35, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_max = 100, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 80, + .ppt_pl3_fppt_max = 125, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 28, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 54, + .ppt_pl2_sppt_def = 40, + .ppt_pl2_sppt_max = 60, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 80, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GV601V"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_def = 100, + .ppt_pl1_spl_max = 110, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 40, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 40, + .ppt_pl2_sppt_max = 60, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "GX650P"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 110, + .ppt_pl1_spl_max = 130, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 125, + .ppt_pl2_sppt_max = 130, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 125, + .ppt_pl3_fppt_max = 135, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_def = 25, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_def = 35, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_def = 42, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G513I"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + /* Yes this laptop is very limited */ + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 80, + }, + .dc_data = NULL, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G513QM"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + /* Yes this laptop is very limited */ + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 100, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 190, + }, + .dc_data = NULL, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G513R"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 35, + .ppt_pl1_spl_max = 90, + .ppt_pl2_sppt_min = 54, + .ppt_pl2_sppt_max = 100, + .ppt_pl3_fppt_min = 54, + .ppt_pl3_fppt_max = 125, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 50, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 50, + .ppt_pl3_fppt_min = 28, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G614J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G634J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G733C"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 170, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 35, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G733P"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 30, + .ppt_pl1_spl_def = 100, + .ppt_pl1_spl_max = 130, + .ppt_pl2_sppt_min = 65, + .ppt_pl2_sppt_def = 125, + .ppt_pl2_sppt_max = 130, + .ppt_pl3_fppt_min = 65, + .ppt_pl3_fppt_def = 125, + .ppt_pl3_fppt_max = 130, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 65, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 65, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 75, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G814J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 140, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "G834J"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 28, + .ppt_pl1_spl_max = 140, + .ppt_pl2_sppt_min = 28, + .ppt_pl2_sppt_max = 175, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 25, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 55, + .ppt_pl2_sppt_min = 25, + .ppt_pl2_sppt_max = 70, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + }, + .requires_fan_curve = true, + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "H7606W"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 15, + .ppt_pl1_spl_max = 80, + .ppt_pl2_sppt_min = 35, + .ppt_pl2_sppt_max = 80, + .ppt_pl3_fppt_min = 35, + .ppt_pl3_fppt_max = 80, + .nv_dynamic_boost_min = 5, + .nv_dynamic_boost_max = 20, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + .nv_tgp_min = 55, + .nv_tgp_max = 85, + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 25, + .ppt_pl1_spl_max = 35, + .ppt_pl2_sppt_min = 31, + .ppt_pl2_sppt_max = 44, + .ppt_pl3_fppt_min = 45, + .ppt_pl3_fppt_max = 65, + .nv_temp_target_min = 75, + .nv_temp_target_max = 87, + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "RC71"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_max = 30, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 43, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_max = 53 + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_def = 15, + .ppt_pl1_spl_max = 25, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_def = 20, + .ppt_pl2_sppt_max = 30, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_def = 25, + .ppt_pl3_fppt_max = 35 + } + }, + }, + { + .matches = { + DMI_MATCH(DMI_BOARD_NAME, "RC72"), + }, + .driver_data = &(struct power_data) { + .ac_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_max = 30, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_max = 43, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_max = 53 + }, + .dc_data = &(struct power_limits) { + .ppt_pl1_spl_min = 7, + .ppt_pl1_spl_def = 17, + .ppt_pl1_spl_max = 25, + .ppt_pl2_sppt_min = 15, + .ppt_pl2_sppt_def = 24, + .ppt_pl2_sppt_max = 30, + .ppt_pl3_fppt_min = 15, + .ppt_pl3_fppt_def = 30, + .ppt_pl3_fppt_max = 35 + } + }, + }, + {} +}; + +#endif /* _ASUS_ARMOURY_H_ */ diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 47cc766624d7..942f1013d6cb 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -55,8 +55,6 @@ module_param(fnlock_default, bool, 0444); #define to_asus_wmi_driver(pdrv) \ (container_of((pdrv), struct asus_wmi_driver, platform_driver)) -#define ASUS_WMI_MGMT_GUID "97845ED0-4E6D-11DE-8A39-0800200C9A66" - #define NOTIFY_BRNUP_MIN 0x11 #define NOTIFY_BRNUP_MAX 0x1f #define NOTIFY_BRNDOWN_MIN 0x20 @@ -105,8 +103,6 @@ module_param(fnlock_default, bool, 0444); #define USB_INTEL_XUSB2PR 0xD0 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI 0x9c31 -#define ASUS_ACPI_UID_ASUSWMI "ASUSWMI" - #define WMI_EVENT_MASK 0xFFFF #define FAN_CURVE_POINTS 8 @@ -127,7 +123,6 @@ module_param(fnlock_default, bool, 0444); #define NVIDIA_TEMP_MIN 75 #define NVIDIA_TEMP_MAX 87 -#define ASUS_SCREENPAD_BRIGHT_MIN 20 #define ASUS_SCREENPAD_BRIGHT_MAX 255 #define ASUS_SCREENPAD_BRIGHT_DEFAULT 60 @@ -142,16 +137,20 @@ module_param(fnlock_default, bool, 0444); #define ASUS_MINI_LED_2024_STRONG 0x01 #define ASUS_MINI_LED_2024_OFF 0x02 -/* Controls the power state of the USB0 hub on ROG Ally which input is on */ #define ASUS_USB0_PWR_EC0_CSEE "\\_SB.PCI0.SBRG.EC0.CSEE" -/* 300ms so far seems to produce a reliable result on AC and battery */ -#define ASUS_USB0_PWR_EC0_CSEE_WAIT 1500 +/* + * The period required to wait after screen off/on/s2idle.check in MS. + * Time here greatly impacts the wake behaviour. Used in suspend/wake. + */ +#define ASUS_USB0_PWR_EC0_CSEE_WAIT 600 +#define ASUS_USB0_PWR_EC0_CSEE_OFF 0xB7 +#define ASUS_USB0_PWR_EC0_CSEE_ON 0xB8 static const char * const ashs_ids[] = { "ATK4001", "ATK4002", NULL }; static int throttle_thermal_policy_write(struct asus_wmi *); -static const struct dmi_system_id asus_ally_mcu_quirk[] = { +static const struct dmi_system_id asus_rog_ally_device[] = { { .matches = { DMI_MATCH(DMI_BOARD_NAME, "RC71L"), @@ -274,9 +273,6 @@ struct asus_wmi { u32 tablet_switch_dev_id; bool tablet_switch_inverted; - /* The ROG Ally device requires the MCU USB device be disconnected before suspend */ - bool ally_mcu_usb_switch; - enum fan_type fan_type; enum fan_type gpu_fan_type; enum fan_type mid_fan_type; @@ -336,6 +332,16 @@ struct asus_wmi { struct asus_wmi_driver *driver; }; +/* Global to allow setting externally without requiring driver data */ +static enum asus_ally_mcu_hack use_ally_mcu_hack = ASUS_WMI_ALLY_MCU_HACK_INIT; + +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) +static void asus_wmi_show_deprecated(void) +{ + pr_notice_once("Accessing attributes through /sys/bus/platform/asus_wmi is deprecated and will be removed in a future release. Please switch over to /sys/class/firmware_attributes.\n"); +} +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ + /* WMI ************************************************************************/ static int asus_wmi_evaluate_method3(u32 method_id, @@ -386,7 +392,7 @@ int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval) { return asus_wmi_evaluate_method3(method_id, arg0, arg1, 0, retval); } -EXPORT_SYMBOL_GPL(asus_wmi_evaluate_method); +EXPORT_SYMBOL_NS_GPL(asus_wmi_evaluate_method, "ASUS_WMI"); static int asus_wmi_evaluate_method5(u32 method_id, u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4, u32 *retval) @@ -550,12 +556,51 @@ static int asus_wmi_get_devstate(struct asus_wmi *asus, u32 dev_id, u32 *retval) return 0; } -static int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, - u32 *retval) + +/** + * asus_wmi_get_devstate_dsts() - Get the WMI function state. + * @dev_id: The WMI method ID to call. + * @retval: A pointer to where to store the value returned from WMI. + * + * On success the return value is 0, and the retval is a valid value returned + * by the successful WMI function call otherwise an error is returned if the + * call failed, or if the WMI method ID is unsupported. + */ +int asus_wmi_get_devstate_dsts(u32 dev_id, u32 *retval) +{ + int err; + + err = asus_wmi_evaluate_method(ASUS_WMI_METHODID_DSTS, dev_id, 0, retval); + if (err) + return err; + + if (*retval == ASUS_WMI_UNSUPPORTED_METHOD) + return -ENODEV; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(asus_wmi_get_devstate_dsts, "ASUS_WMI"); + +/** + * asus_wmi_set_devstate() - Set the WMI function state. + * @dev_id: The WMI function to call. + * @ctrl_param: The argument to be used for this WMI function. + * @retval: A pointer to where to store the value returned from WMI. + * + * The returned WMI function state if not checked here for error as + * asus_wmi_set_devstate() is not called unless first paired with a call to + * asus_wmi_get_devstate_dsts() to check that the WMI function is supported. + * + * On success the return value is 0, and the retval is a valid value returned + * by the successful WMI function call. An error value is returned only if the + * WMI function failed. + */ +int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval) { return asus_wmi_evaluate_method(ASUS_WMI_METHODID_DEVS, dev_id, ctrl_param, retval); } +EXPORT_SYMBOL_NS_GPL(asus_wmi_set_devstate, "ASUS_WMI"); /* Helper for special devices with magic return codes */ static int asus_wmi_get_devstate_bits(struct asus_wmi *asus, @@ -688,6 +733,7 @@ static void asus_wmi_tablet_mode_get_state(struct asus_wmi *asus) } /* Charging mode, 1=Barrel, 2=USB ******************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t charge_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -698,12 +744,16 @@ static ssize_t charge_mode_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", value & 0xff); } static DEVICE_ATTR_RO(charge_mode); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* dGPU ********************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t dgpu_disable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -714,6 +764,8 @@ static ssize_t dgpu_disable_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -767,8 +819,10 @@ static ssize_t dgpu_disable_store(struct device *dev, return count; } static DEVICE_ATTR_RW(dgpu_disable); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* eGPU ********************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t egpu_enable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -779,6 +833,8 @@ static ssize_t egpu_enable_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -835,8 +891,10 @@ static ssize_t egpu_enable_store(struct device *dev, return count; } static DEVICE_ATTR_RW(egpu_enable); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Is eGPU connected? *********************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t egpu_connected_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -847,12 +905,16 @@ static ssize_t egpu_connected_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } static DEVICE_ATTR_RO(egpu_connected); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* gpu mux switch *************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t gpu_mux_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -863,6 +925,8 @@ static ssize_t gpu_mux_mode_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -921,6 +985,7 @@ static ssize_t gpu_mux_mode_store(struct device *dev, return count; } static DEVICE_ATTR_RW(gpu_mux_mode); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* TUF Laptop Keyboard RGB Modes **********************************************/ static ssize_t kbd_rgb_mode_store(struct device *dev, @@ -1044,6 +1109,7 @@ static const struct attribute_group *kbd_rgb_mode_groups[] = { }; /* Tunable: PPT: Intel=PL1, AMD=SPPT *****************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t ppt_pl2_sppt_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -1082,6 +1148,8 @@ static ssize_t ppt_pl2_sppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_pl2_sppt); } static DEVICE_ATTR_RW(ppt_pl2_sppt); @@ -1124,6 +1192,8 @@ static ssize_t ppt_pl1_spl_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_pl1_spl); } static DEVICE_ATTR_RW(ppt_pl1_spl); @@ -1167,6 +1237,8 @@ static ssize_t ppt_fppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_fppt); } static DEVICE_ATTR_RW(ppt_fppt); @@ -1210,6 +1282,8 @@ static ssize_t ppt_apu_sppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_apu_sppt); } static DEVICE_ATTR_RW(ppt_apu_sppt); @@ -1253,6 +1327,8 @@ static ssize_t ppt_platform_sppt_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->ppt_platform_sppt); } static DEVICE_ATTR_RW(ppt_platform_sppt); @@ -1296,6 +1372,8 @@ static ssize_t nv_dynamic_boost_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->nv_dynamic_boost); } static DEVICE_ATTR_RW(nv_dynamic_boost); @@ -1339,11 +1417,53 @@ static ssize_t nv_temp_target_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%u\n", asus->nv_temp_target); } static DEVICE_ATTR_RW(nv_temp_target); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Ally MCU Powersave ********************************************************/ + +/* + * The HID driver needs to check MCU version and set this to false if the MCU FW + * version is >= the minimum requirements. New FW do not need the hacks. + */ +void set_ally_mcu_hack(enum asus_ally_mcu_hack status) +{ + use_ally_mcu_hack = status; + pr_debug("%s Ally MCU suspend quirk\n", + status == ASUS_WMI_ALLY_MCU_HACK_ENABLED ? "Enabled" : "Disabled"); +} +EXPORT_SYMBOL_NS_GPL(set_ally_mcu_hack, "ASUS_WMI"); + +/* + * mcu_powersave should be enabled always, as it is fixed in MCU FW versions: + * - v313 for Ally X + * - v319 for Ally 1 + * The HID driver checks MCU versions and so should set this if requirements match + */ +void set_ally_mcu_powersave(bool enabled) +{ + int result, err; + + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_MCU_POWERSAVE, enabled, &result); + if (err) { + pr_warn("Failed to set MCU powersave: %d\n", err); + return; + } + if (result > 1) { + pr_warn("Failed to set MCU powersave (result): 0x%x\n", result); + return; + } + + pr_debug("%s MCU Powersave\n", + enabled ? "Enabled" : "Disabled"); +} +EXPORT_SYMBOL_NS_GPL(set_ally_mcu_powersave, "ASUS_WMI"); + +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t mcu_powersave_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1354,6 +1474,8 @@ static ssize_t mcu_powersave_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -1389,6 +1511,7 @@ static ssize_t mcu_powersave_store(struct device *dev, return count; } static DEVICE_ATTR_RW(mcu_powersave); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Battery ********************************************************************/ @@ -2262,6 +2385,7 @@ static int asus_wmi_rfkill_init(struct asus_wmi *asus) } /* Panel Overdrive ************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t panel_od_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2272,6 +2396,8 @@ static ssize_t panel_od_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -2308,9 +2434,10 @@ static ssize_t panel_od_store(struct device *dev, return count; } static DEVICE_ATTR_RW(panel_od); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Bootup sound ***************************************************************/ - +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t boot_sound_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2321,6 +2448,8 @@ static ssize_t boot_sound_show(struct device *dev, if (result < 0) return result; + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", result); } @@ -2356,8 +2485,10 @@ static ssize_t boot_sound_store(struct device *dev, return count; } static DEVICE_ATTR_RW(boot_sound); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Mini-LED mode **************************************************************/ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t mini_led_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2388,6 +2519,8 @@ static ssize_t mini_led_mode_show(struct device *dev, } } + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "%d\n", value); } @@ -2458,10 +2591,13 @@ static ssize_t available_mini_led_mode_show(struct device *dev, return sysfs_emit(buf, "0 1 2\n"); } + asus_wmi_show_deprecated(); + return sysfs_emit(buf, "0\n"); } static DEVICE_ATTR_RO(available_mini_led_mode); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Quirks *********************************************************************/ @@ -3749,6 +3885,7 @@ static int throttle_thermal_policy_set_default(struct asus_wmi *asus) return throttle_thermal_policy_write(asus); } +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) static ssize_t throttle_thermal_policy_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -3792,6 +3929,7 @@ static ssize_t throttle_thermal_policy_store(struct device *dev, * Throttle thermal policy: 0 - default, 1 - overboost, 2 - silent */ static DEVICE_ATTR_RW(throttle_thermal_policy); +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ /* Platform profile ***********************************************************/ static int asus_wmi_platform_profile_get(struct device *dev, @@ -3811,7 +3949,7 @@ static int asus_wmi_platform_profile_get(struct device *dev, *profile = PLATFORM_PROFILE_PERFORMANCE; break; case ASUS_THROTTLE_THERMAL_POLICY_SILENT: - *profile = PLATFORM_PROFILE_QUIET; + *profile = PLATFORM_PROFILE_LOW_POWER; break; default: return -EINVAL; @@ -3835,7 +3973,7 @@ static int asus_wmi_platform_profile_set(struct device *dev, case PLATFORM_PROFILE_BALANCED: tp = ASUS_THROTTLE_THERMAL_POLICY_DEFAULT; break; - case PLATFORM_PROFILE_QUIET: + case PLATFORM_PROFILE_LOW_POWER: tp = ASUS_THROTTLE_THERMAL_POLICY_SILENT; break; default: @@ -3848,7 +3986,7 @@ static int asus_wmi_platform_profile_set(struct device *dev, static int asus_wmi_platform_profile_probe(void *drvdata, unsigned long *choices) { - set_bit(PLATFORM_PROFILE_QUIET, choices); + set_bit(PLATFORM_PROFILE_LOW_POWER, choices); set_bit(PLATFORM_PROFILE_BALANCED, choices); set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); @@ -4101,43 +4239,37 @@ static int read_screenpad_brightness(struct backlight_device *bd) return err; /* The device brightness can only be read if powered, so return stored */ if (err == BACKLIGHT_POWER_OFF) - return asus->driver->screenpad_brightness - ASUS_SCREENPAD_BRIGHT_MIN; + return bd->props.brightness; err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_SCREENPAD_LIGHT, &retval); if (err < 0) return err; - return (retval & ASUS_WMI_DSTS_BRIGHTNESS_MASK) - ASUS_SCREENPAD_BRIGHT_MIN; + return (retval & ASUS_WMI_DSTS_BRIGHTNESS_MASK); } static int update_screenpad_bl_status(struct backlight_device *bd) { - struct asus_wmi *asus = bl_get_data(bd); - int power, err = 0; + int err = 0; u32 ctrl_param; - power = read_screenpad_backlight_power(asus); - if (power < 0) - return power; - - if (bd->props.power != power) { - if (power != BACKLIGHT_POWER_ON) { - /* Only brightness > 0 can power it back on */ - ctrl_param = asus->driver->screenpad_brightness - ASUS_SCREENPAD_BRIGHT_MIN; - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_LIGHT, - ctrl_param, NULL); - } else { - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 0, NULL); - } - } else if (power == BACKLIGHT_POWER_ON) { - /* Only set brightness if powered on or we get invalid/unsync state */ - ctrl_param = bd->props.brightness + ASUS_SCREENPAD_BRIGHT_MIN; + ctrl_param = bd->props.brightness; + if (ctrl_param >= 0 && bd->props.power) { + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 1, + NULL); + if (err < 0) + return err; + ctrl_param = bd->props.brightness; err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_LIGHT, ctrl_param, NULL); + if (err < 0) + return err; } - /* Ensure brightness is stored to turn back on with */ - if (err == 0) - asus->driver->screenpad_brightness = bd->props.brightness + ASUS_SCREENPAD_BRIGHT_MIN; + if (!bd->props.power) { + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 0, NULL); + if (err < 0) + return err; + } return err; } @@ -4155,22 +4287,19 @@ static int asus_screenpad_init(struct asus_wmi *asus) int err, power; int brightness = 0; - power = read_screenpad_backlight_power(asus); + power = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_SCREENPAD_POWER); if (power < 0) return power; - if (power != BACKLIGHT_POWER_OFF) { + if (power) { err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_SCREENPAD_LIGHT, &brightness); if (err < 0) return err; } - /* default to an acceptable min brightness on boot if too low */ - if (brightness < ASUS_SCREENPAD_BRIGHT_MIN) - brightness = ASUS_SCREENPAD_BRIGHT_DEFAULT; memset(&props, 0, sizeof(struct backlight_properties)); props.type = BACKLIGHT_RAW; /* ensure this bd is last to be picked */ - props.max_brightness = ASUS_SCREENPAD_BRIGHT_MAX - ASUS_SCREENPAD_BRIGHT_MIN; + props.max_brightness = ASUS_SCREENPAD_BRIGHT_MAX; bd = backlight_device_register("asus_screenpad", &asus->platform_device->dev, asus, &asus_screenpad_bl_ops, &props); @@ -4181,7 +4310,7 @@ static int asus_screenpad_init(struct asus_wmi *asus) asus->screenpad_backlight_device = bd; asus->driver->screenpad_brightness = brightness; - bd->props.brightness = brightness - ASUS_SCREENPAD_BRIGHT_MIN; + bd->props.brightness = brightness; bd->props.power = power; backlight_update_status(bd); @@ -4393,27 +4522,29 @@ static struct attribute *platform_attributes[] = { &dev_attr_camera.attr, &dev_attr_cardr.attr, &dev_attr_touchpad.attr, - &dev_attr_charge_mode.attr, - &dev_attr_egpu_enable.attr, - &dev_attr_egpu_connected.attr, - &dev_attr_dgpu_disable.attr, - &dev_attr_gpu_mux_mode.attr, &dev_attr_lid_resume.attr, &dev_attr_als_enable.attr, &dev_attr_fan_boost_mode.attr, - &dev_attr_throttle_thermal_policy.attr, - &dev_attr_ppt_pl2_sppt.attr, - &dev_attr_ppt_pl1_spl.attr, - &dev_attr_ppt_fppt.attr, - &dev_attr_ppt_apu_sppt.attr, - &dev_attr_ppt_platform_sppt.attr, - &dev_attr_nv_dynamic_boost.attr, - &dev_attr_nv_temp_target.attr, - &dev_attr_mcu_powersave.attr, - &dev_attr_boot_sound.attr, - &dev_attr_panel_od.attr, - &dev_attr_mini_led_mode.attr, - &dev_attr_available_mini_led_mode.attr, +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) + &dev_attr_charge_mode.attr, + &dev_attr_egpu_enable.attr, + &dev_attr_egpu_connected.attr, + &dev_attr_dgpu_disable.attr, + &dev_attr_gpu_mux_mode.attr, + &dev_attr_ppt_pl2_sppt.attr, + &dev_attr_ppt_pl1_spl.attr, + &dev_attr_ppt_fppt.attr, + &dev_attr_ppt_apu_sppt.attr, + &dev_attr_ppt_platform_sppt.attr, + &dev_attr_nv_dynamic_boost.attr, + &dev_attr_nv_temp_target.attr, + &dev_attr_mcu_powersave.attr, + &dev_attr_boot_sound.attr, + &dev_attr_panel_od.attr, + &dev_attr_mini_led_mode.attr, + &dev_attr_available_mini_led_mode.attr, + &dev_attr_throttle_thermal_policy.attr, +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ NULL }; @@ -4435,7 +4566,11 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj, devid = ASUS_WMI_DEVID_LID_RESUME; else if (attr == &dev_attr_als_enable.attr) devid = ASUS_WMI_DEVID_ALS_ENABLE; - else if (attr == &dev_attr_charge_mode.attr) + else if (attr == &dev_attr_fan_boost_mode.attr) + ok = asus->fan_boost_mode_available; + +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) + if (attr == &dev_attr_charge_mode.attr) devid = ASUS_WMI_DEVID_CHARGE_MODE; else if (attr == &dev_attr_egpu_enable.attr) ok = asus->egpu_enable_available; @@ -4473,6 +4608,7 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj, ok = asus->mini_led_dev_id != 0; else if (attr == &dev_attr_available_mini_led_mode.attr) ok = asus->mini_led_dev_id != 0; +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ if (devid != -1) { ok = !(asus_wmi_get_devstate_simple(asus, devid) < 0); @@ -4712,7 +4848,23 @@ static int asus_wmi_add(struct platform_device *pdev) if (err) goto fail_platform; + if (use_ally_mcu_hack == ASUS_WMI_ALLY_MCU_HACK_INIT) { + if (acpi_has_method(NULL, ASUS_USB0_PWR_EC0_CSEE) + && dmi_check_system(asus_rog_ally_device)) + use_ally_mcu_hack = ASUS_WMI_ALLY_MCU_HACK_ENABLED; + if (dmi_match(DMI_BOARD_NAME, "RC71")) { + /* + * These steps ensure the device is in a valid good state, this is + * especially important for the Ally 1 after a reboot. + */ + acpi_execute_simple_method(NULL, ASUS_USB0_PWR_EC0_CSEE, + ASUS_USB0_PWR_EC0_CSEE_ON); + msleep(ASUS_USB0_PWR_EC0_CSEE_WAIT); + } + } + /* ensure defaults for tunables */ +#if IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) asus->ppt_pl2_sppt = 5; asus->ppt_pl1_spl = 5; asus->ppt_apu_sppt = 5; @@ -4725,8 +4877,6 @@ static int asus_wmi_add(struct platform_device *pdev) asus->dgpu_disable_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_DGPU); asus->kbd_rgb_state_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_STATE); asus->oobe_state_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_OOBE); - asus->ally_mcu_usb_switch = acpi_has_method(NULL, ASUS_USB0_PWR_EC0_CSEE) - && dmi_check_system(asus_ally_mcu_quirk); if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_MINI_LED_MODE)) asus->mini_led_dev_id = ASUS_WMI_DEVID_MINI_LED_MODE; @@ -4737,17 +4887,18 @@ static int asus_wmi_add(struct platform_device *pdev) asus->gpu_mux_dev = ASUS_WMI_DEVID_GPU_MUX; else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_GPU_MUX_VIVO)) asus->gpu_mux_dev = ASUS_WMI_DEVID_GPU_MUX_VIVO; - - if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE)) - asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE; - else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE2)) - asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE2; +#endif /* IS_ENABLED(CONFIG_ASUS_WMI_DEPRECATED_ATTRS) */ if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY)) asus->throttle_thermal_policy_dev = ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY; else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY_VIVO)) asus->throttle_thermal_policy_dev = ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY_VIVO; + if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE)) + asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE; + else if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE2)) + asus->kbd_rgb_dev = ASUS_WMI_DEVID_TUF_RGB_MODE2; + err = fan_boost_mode_check_present(asus); if (err) goto fail_fan_boost_mode; @@ -4913,34 +5064,6 @@ static int asus_hotk_resume(struct device *device) return 0; } -static int asus_hotk_resume_early(struct device *device) -{ - struct asus_wmi *asus = dev_get_drvdata(device); - - if (asus->ally_mcu_usb_switch) { - /* sleep required to prevent USB0 being yanked then reappearing rapidly */ - if (ACPI_FAILURE(acpi_execute_simple_method(NULL, ASUS_USB0_PWR_EC0_CSEE, 0xB8))) - dev_err(device, "ROG Ally MCU failed to connect USB dev\n"); - else - msleep(ASUS_USB0_PWR_EC0_CSEE_WAIT); - } - return 0; -} - -static int asus_hotk_prepare(struct device *device) -{ - struct asus_wmi *asus = dev_get_drvdata(device); - - if (asus->ally_mcu_usb_switch) { - /* sleep required to ensure USB0 is disabled before sleep continues */ - if (ACPI_FAILURE(acpi_execute_simple_method(NULL, ASUS_USB0_PWR_EC0_CSEE, 0xB7))) - dev_err(device, "ROG Ally MCU failed to disconnect USB dev\n"); - else - msleep(ASUS_USB0_PWR_EC0_CSEE_WAIT); - } - return 0; -} - static int asus_hotk_restore(struct device *device) { struct asus_wmi *asus = dev_get_drvdata(device); @@ -4988,11 +5111,34 @@ static int asus_hotk_restore(struct device *device) return 0; } +static void asus_ally_s2idle_restore(void) +{ + if (use_ally_mcu_hack == ASUS_WMI_ALLY_MCU_HACK_ENABLED) { + acpi_execute_simple_method(NULL, ASUS_USB0_PWR_EC0_CSEE, + ASUS_USB0_PWR_EC0_CSEE_ON); + msleep(ASUS_USB0_PWR_EC0_CSEE_WAIT); + } +} + +static int asus_hotk_prepare(struct device *device) +{ + if (use_ally_mcu_hack == ASUS_WMI_ALLY_MCU_HACK_ENABLED) { + acpi_execute_simple_method(NULL, ASUS_USB0_PWR_EC0_CSEE, + ASUS_USB0_PWR_EC0_CSEE_OFF); + msleep(ASUS_USB0_PWR_EC0_CSEE_WAIT); + } + return 0; +} + +/* Use only for Ally devices due to the wake_on_ac */ +static struct acpi_s2idle_dev_ops asus_ally_s2idle_dev_ops = { + .restore = asus_ally_s2idle_restore, +}; + static const struct dev_pm_ops asus_pm_ops = { .thaw = asus_hotk_thaw, .restore = asus_hotk_restore, .resume = asus_hotk_resume, - .resume_early = asus_hotk_resume_early, .prepare = asus_hotk_prepare, }; @@ -5020,6 +5166,10 @@ static int asus_wmi_probe(struct platform_device *pdev) return ret; } + ret = acpi_register_lps0_dev(&asus_ally_s2idle_dev_ops); + if (ret) + pr_warn("failed to register LPS0 sleep handler in asus-wmi\n"); + return asus_wmi_add(pdev); } @@ -5052,6 +5202,7 @@ EXPORT_SYMBOL_GPL(asus_wmi_register_driver); void asus_wmi_unregister_driver(struct asus_wmi_driver *driver) { + acpi_unregister_lps0_dev(&asus_ally_s2idle_dev_ops); platform_device_unregister(driver->platform_device); platform_driver_unregister(&driver->platform_driver); used = false; diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 783e2a336861..78261ea49995 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -6,6 +6,9 @@ #include #include +#define ASUS_WMI_MGMT_GUID "97845ED0-4E6D-11DE-8A39-0800200C9A66" +#define ASUS_ACPI_UID_ASUSWMI "ASUSWMI" + /* WMI Methods */ #define ASUS_WMI_METHODID_SPEC 0x43455053 /* BIOS SPECification */ #define ASUS_WMI_METHODID_SFBD 0x44424653 /* Set First Boot Device */ @@ -73,12 +76,14 @@ #define ASUS_WMI_DEVID_THROTTLE_THERMAL_POLICY_VIVO 0x00110019 /* Misc */ +#define ASUS_WMI_DEVID_PANEL_HD 0x0005001C #define ASUS_WMI_DEVID_PANEL_OD 0x00050019 #define ASUS_WMI_DEVID_CAMERA 0x00060013 #define ASUS_WMI_DEVID_LID_FLIP 0x00060062 #define ASUS_WMI_DEVID_LID_FLIP_ROG 0x00060077 #define ASUS_WMI_DEVID_MINI_LED_MODE 0x0005001E #define ASUS_WMI_DEVID_MINI_LED_MODE2 0x0005002E +#define ASUS_WMI_DEVID_SCREEN_AUTO_BRIGHTNESS 0x0005002A /* Storage */ #define ASUS_WMI_DEVID_CARDREADER 0x00080013 @@ -133,6 +138,16 @@ /* dgpu on/off */ #define ASUS_WMI_DEVID_DGPU 0x00090020 +/* Intel E-core and P-core configuration in a format 0x0[E]0[P] */ +#define ASUS_WMI_DEVID_CORES 0x001200D2 + /* Maximum Intel E-core and P-core availability */ +#define ASUS_WMI_DEVID_CORES_MAX 0x001200D3 + +#define ASUS_WMI_DEVID_APU_MEM 0x000600C1 + +#define ASUS_WMI_DEVID_DGPU_BASE_TGP 0x00120099 +#define ASUS_WMI_DEVID_DGPU_SET_TGP 0x00120098 + /* gpu mux switch, 0 = dGPU, 1 = Optimus */ #define ASUS_WMI_DEVID_GPU_MUX 0x00090016 #define ASUS_WMI_DEVID_GPU_MUX_VIVO 0x00090026 @@ -157,9 +172,37 @@ #define ASUS_WMI_DSTS_MAX_BRIGTH_MASK 0x0000FF00 #define ASUS_WMI_DSTS_LIGHTBAR_MASK 0x0000000F +enum asus_ally_mcu_hack { + ASUS_WMI_ALLY_MCU_HACK_INIT, + ASUS_WMI_ALLY_MCU_HACK_ENABLED, + ASUS_WMI_ALLY_MCU_HACK_DISABLED, +}; + #if IS_REACHABLE(CONFIG_ASUS_WMI) +void set_ally_mcu_hack(enum asus_ally_mcu_hack status); +void set_ally_mcu_powersave(bool enabled); +int asus_wmi_get_devstate_dsts(u32 dev_id, u32 *retval); +int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval); int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval); #else +static inline void set_ally_mcu_hack(enum asus_ally_mcu_hack status) +{ +} +static inline void set_ally_mcu_powersave(bool enabled) +{ +} +static inline int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval) +{ + return -ENODEV; +} +static inline int asus_wmi_get_devstate_dsts(u32 dev_id, u32 *retval) +{ + return -ENODEV; +} +static inline int asus_wmi_set_devstate(u32 dev_id, u32 ctrl_param, u32 *retval) +{ + return -ENODEV; +} static inline int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval) { -- 2.49.0.654.g845c48a16a From 046dc1d355f5de907bcecc100c501d8871d56427 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:55:07 +0200 Subject: [PATCH 3/8] async-shutdown Signed-off-by: Peter Jung --- drivers/base/base.h | 8 +- drivers/base/core.c | 182 ++++++++++++++++++++++++++++------ drivers/nvme/host/pci.c | 1 + include/linux/device/driver.h | 2 + 4 files changed, 161 insertions(+), 32 deletions(-) diff --git a/drivers/base/base.h b/drivers/base/base.h index 123031a757d9..4b486c23be31 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -10,6 +10,7 @@ * shared outside of the drivers/base/ directory. * */ +#include #include /** @@ -98,11 +99,14 @@ struct driver_private { * the device; typically because it depends on another driver getting * probed first. * @async_driver - pointer to device driver awaiting probe via async_probe + * @shutdown_after - used during device shutdown to ensure correct shutdown + * ordering. * @device - pointer back to the struct device that this structure is * associated with. * @dead - This device is currently either in the process of or has been * removed from the system. Any asynchronous events scheduled for this * device should exit without taking any action. + * @async_shutdown_queued - indicates async shutdown is enqueued for this device * * Nothing outside of the driver core should ever touch these fields. */ @@ -115,8 +119,10 @@ struct device_private { struct list_head deferred_probe; const struct device_driver *async_driver; char *deferred_probe_reason; + async_cookie_t shutdown_after; struct device *device; - u8 dead:1; + u8 dead:1, + async_shutdown_queued:1; }; #define to_device_private_parent(obj) \ container_of(obj, struct device_private, knode_parent) diff --git a/drivers/base/core.c b/drivers/base/core.c index cbc0099d8ef2..6f1b550f3aa6 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -4786,18 +4787,156 @@ int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid) } EXPORT_SYMBOL_GPL(device_change_owner); +static ASYNC_DOMAIN(sd_domain); + +static void shutdown_one_device(struct device *dev) +{ + /* hold lock to avoid race with probe/release */ + if (dev->parent && dev->bus && dev->bus->need_parent_lock) + device_lock(dev->parent); + device_lock(dev); + + /* Don't allow any more runtime suspends */ + pm_runtime_get_noresume(dev); + pm_runtime_barrier(dev); + + if (dev->class && dev->class->shutdown_pre) { + if (initcall_debug) + dev_info(dev, "shutdown_pre\n"); + dev->class->shutdown_pre(dev); + } + if (dev->bus && dev->bus->shutdown) { + if (initcall_debug) + dev_info(dev, "shutdown\n"); + dev->bus->shutdown(dev); + } else if (dev->driver && dev->driver->shutdown) { + if (initcall_debug) + dev_info(dev, "shutdown\n"); + dev->driver->shutdown(dev); + } + + device_unlock(dev); + if (dev->parent && dev->bus && dev->bus->need_parent_lock) + device_unlock(dev->parent); + + put_device(dev); + if (dev->parent) + put_device(dev->parent); +} + +static bool device_wants_async_shutdown(struct device *dev) +{ + if (dev->driver && dev->driver->async_shutdown_enable) + return true; + + return false; +} + +/** + * shutdown_one_device_async + * @data: the pointer to the struct device to be shutdown + * @cookie: not used + * + * Shuts down one device, after waiting for device's last child or consumer to + * be shutdown. + * + * shutdown_after is set to the shutdown cookie of the last child or consumer + * of this device (if any). + */ +static void shutdown_one_device_async(void *data, async_cookie_t cookie) +{ + struct device_private *p = data; + + if (p->shutdown_after) + async_synchronize_cookie_domain(p->shutdown_after, &sd_domain); + + shutdown_one_device(p->device); +} + +static void queue_device_async_shutdown(struct device *dev) +{ + struct device_link *link; + struct device *parent; + async_cookie_t cookie; + int idx; + + parent = get_device(dev->parent); + get_device(dev); + + /* + * Add one to this device's cookie so that when shutdown_after is passed + * to async_synchronize_cookie_domain(), it will wait until *after* + * shutdown_one_device_async() is finished running for this device. + */ + cookie = async_schedule_domain(shutdown_one_device_async, dev->p, + &sd_domain) + 1; + + /* + * Set async_shutdown_queued to avoid overwriting a parent's + * shutdown_after while the parent is shutting down. This can happen if + * a parent or supplier is not ordered in the devices_kset list before a + * child or consumer, which is not expected. + */ + dev->p->async_shutdown_queued = 1; + + /* Ensure any parent & suppliers wait for this device to shut down */ + if (parent) { + if (!parent->p->async_shutdown_queued) + parent->p->shutdown_after = cookie; + put_device(parent); + } + + idx = device_links_read_lock(); + list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, + device_links_read_lock_held()) { + /* + * sync_state_only devlink consumers aren't dependent on + * suppliers + */ + if (!device_link_flag_is_sync_state_only(link->flags) && + !link->supplier->p->async_shutdown_queued) + link->supplier->p->shutdown_after = cookie; + } + device_links_read_unlock(idx); + put_device(dev); +} + /** * device_shutdown - call ->shutdown() on each device to shutdown. */ void device_shutdown(void) { - struct device *dev, *parent; + struct device *dev, *parent, *tmp; + LIST_HEAD(async_list); + bool wait_for_async; wait_for_device_probe(); device_block_probing(); cpufreq_suspend(); + /* + * Find devices which can shut down asynchronously, and move them from + * the devices list onto the async list in reverse order. + */ + spin_lock(&devices_kset->list_lock); + list_for_each_entry_safe(dev, tmp, &devices_kset->list, kobj.entry) { + if (device_wants_async_shutdown(dev)) { + get_device(dev->parent); + get_device(dev); + list_move(&dev->kobj.entry, &async_list); + } + } + spin_unlock(&devices_kset->list_lock); + + /* + * Dispatch asynchronous shutdowns first so they don't have to wait + * behind any synchronous shutdowns. + */ + wait_for_async = !list_empty(&async_list); + list_for_each_entry_safe(dev, tmp, &async_list, kobj.entry) + queue_device_async_shutdown(dev); + spin_lock(&devices_kset->list_lock); /* * Walk the devices list backward, shutting down each in turn. @@ -4822,40 +4961,21 @@ void device_shutdown(void) list_del_init(&dev->kobj.entry); spin_unlock(&devices_kset->list_lock); - /* hold lock to avoid race with probe/release */ - if (parent) - device_lock(parent); - device_lock(dev); - - /* Don't allow any more runtime suspends */ - pm_runtime_get_noresume(dev); - pm_runtime_barrier(dev); - - if (dev->class && dev->class->shutdown_pre) { - if (initcall_debug) - dev_info(dev, "shutdown_pre\n"); - dev->class->shutdown_pre(dev); - } - if (dev->bus && dev->bus->shutdown) { - if (initcall_debug) - dev_info(dev, "shutdown\n"); - dev->bus->shutdown(dev); - } else if (dev->driver && dev->driver->shutdown) { - if (initcall_debug) - dev_info(dev, "shutdown\n"); - dev->driver->shutdown(dev); - } - - device_unlock(dev); - if (parent) - device_unlock(parent); - - put_device(dev); - put_device(parent); + /* + * Dispatch an async shutdown if this device has a child or + * consumer that is async. Otherwise, shut down synchronously. + */ + if (dev->p->shutdown_after) + queue_device_async_shutdown(dev); + else + shutdown_one_device(dev); spin_lock(&devices_kset->list_lock); } spin_unlock(&devices_kset->list_lock); + + if (wait_for_async) + async_synchronize_full_domain(&sd_domain); } /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index f1dd804151b1..8869da839ce6 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -3795,6 +3795,7 @@ static struct pci_driver nvme_driver = { .shutdown = nvme_shutdown, .driver = { .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .async_shutdown_enable = true, #ifdef CONFIG_PM_SLEEP .pm = &nvme_dev_pm_ops, #endif diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index cd8e0f0a634b..c63bc0050c84 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -56,6 +56,7 @@ enum probe_type { * @mod_name: Used for built-in modules. * @suppress_bind_attrs: Disables bind/unbind via sysfs. * @probe_type: Type of the probe (synchronous or asynchronous) to use. + * @async_shutdown_enable: Enables devices to be shutdown asynchronously. * @of_match_table: The open firmware table. * @acpi_match_table: The ACPI match table. * @probe: Called to query the existence of a specific device, @@ -102,6 +103,7 @@ struct device_driver { bool suppress_bind_attrs; /* disables bind/unbind via sysfs */ enum probe_type probe_type; + bool async_shutdown_enable; const struct of_device_id *of_match_table; const struct acpi_device_id *acpi_match_table; -- 2.49.0.654.g845c48a16a From 32e7da4eca82c08c16a17ca8194dc854e7156f3c Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:55:20 +0200 Subject: [PATCH 4/8] bbr3 Signed-off-by: Peter Jung --- include/linux/tcp.h | 6 +- include/net/inet_connection_sock.h | 4 +- include/net/tcp.h | 73 +- include/uapi/linux/inet_diag.h | 23 + include/uapi/linux/rtnetlink.h | 4 +- include/uapi/linux/tcp.h | 1 + net/ipv4/Kconfig | 21 +- net/ipv4/bpf_tcp_ca.c | 4 +- net/ipv4/tcp.c | 3 + net/ipv4/tcp_bbr.c | 2232 +++++++++++++++++++++------- net/ipv4/tcp_cong.c | 1 + net/ipv4/tcp_input.c | 40 +- net/ipv4/tcp_minisocks.c | 2 + net/ipv4/tcp_output.c | 48 +- net/ipv4/tcp_rate.c | 30 +- net/ipv4/tcp_timer.c | 4 +- 16 files changed, 1941 insertions(+), 555 deletions(-) diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 1669d95bb0f9..951a5ed55a27 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -248,7 +248,8 @@ struct tcp_sock { void (*tcp_clean_acked)(struct sock *sk, u32 acked_seq); #endif u32 snd_ssthresh; /* Slow start size threshold */ - u8 recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ + u32 recvmsg_inq : 1,/* Indicate # of bytes in queue upon recvmsg */ + fast_ack_mode:1;/* ack ASAP if >1 rcv_mss received? */ __cacheline_group_end(tcp_sock_read_rx); /* TX read-write hotpath cache lines */ @@ -305,7 +306,8 @@ struct tcp_sock { */ struct tcp_options_received rx_opt; u8 nonagle : 4,/* Disable Nagle algorithm? */ - rate_app_limited:1; /* rate_{delivered,interval_us} limited? */ + rate_app_limited:1, /* rate_{delivered,interval_us} limited? */ + tlp_orig_data_app_limited:1; /* app-limited before TLP rtx? */ __cacheline_group_end(tcp_sock_write_txrx); /* RX read-write hotpath cache lines */ diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 1735db332aab..2c4a94af7093 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -132,8 +132,8 @@ struct inet_connection_sock { u32 icsk_probes_tstamp; u32 icsk_user_timeout; - u64 icsk_ca_priv[104 / sizeof(u64)]; -#define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv) +#define ICSK_CA_PRIV_SIZE (144) + u64 icsk_ca_priv[ICSK_CA_PRIV_SIZE / sizeof(u64)]; }; #define ICSK_TIME_RETRANS 1 /* Retransmit timer */ diff --git a/include/net/tcp.h b/include/net/tcp.h index 4450c384ef17..61f73ca30be3 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -379,11 +379,14 @@ static inline void tcp_dec_quickack_mode(struct sock *sk) #define TCP_ECN_DEMAND_CWR BIT(2) #define TCP_ECN_SEEN BIT(3) #define TCP_ECN_MODE_ACCECN BIT(4) +#define TCP_ECN_LOW BIT(5) +#define TCP_ECN_ECT_PERMANENT BIT(6) #define TCP_ECN_DISABLED 0 #define TCP_ECN_MODE_PENDING (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) #define TCP_ECN_MODE_ANY (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) + static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp) { return tp->ecn_flags & TCP_ECN_MODE_ANY; @@ -840,6 +843,15 @@ static inline void tcp_fast_path_check(struct sock *sk) u32 tcp_delack_max(const struct sock *sk); +static inline void tcp_set_ecn_low_from_dst(struct sock *sk, + const struct dst_entry *dst) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (dst_feature(dst, RTAX_FEATURE_ECN_LOW)) + tp->ecn_flags |= TCP_ECN_LOW; +} + /* Compute the actual rto_min value */ static inline u32 tcp_rto_min(const struct sock *sk) { @@ -945,6 +957,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) return max_t(s64, t1 - t0, 0); } +static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0) +{ + return max_t(s32, t1 - t0, 0); +} + /* provide the departure time in us unit */ static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) { @@ -1043,9 +1060,14 @@ struct tcp_skb_cb { /* pkts S/ACKed so far upon tx of skb, incl retrans: */ __u32 delivered; /* start of send pipeline phase */ - u64 first_tx_mstamp; + u32 first_tx_mstamp; /* when we reached the "delivered" count */ - u64 delivered_mstamp; + u32 delivered_mstamp; +#define TCPCB_IN_FLIGHT_BITS 20 +#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1) + u32 in_flight:20, /* packets in flight at transmit */ + unused2:12; + u32 lost; /* packets lost so far upon tx of skb */ } tx; /* only used for outgoing skbs */ union { struct inet_skb_parm h4; @@ -1158,6 +1180,7 @@ enum tcp_ca_event { CA_EVENT_LOSS, /* loss timeout */ CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ + CA_EVENT_TLP_RECOVERY, /* a lost segment was repaired by TLP probe */ }; /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ @@ -1180,7 +1203,11 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */ +#define TCP_CONG_WANTS_CE_EVENTS BIT(2) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \ + TCP_CONG_NEEDS_ECN | \ + TCP_CONG_WANTS_CE_EVENTS) union tcp_cc_info; @@ -1200,10 +1227,13 @@ struct ack_sample { */ struct rate_sample { u64 prior_mstamp; /* starting timestamp for interval */ + u32 prior_lost; /* tp->lost at "prior_mstamp" */ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ + u32 tx_in_flight; /* packets in flight at starting timestamp */ + s32 lost; /* number of packets lost over interval */ s32 delivered; /* number of packets delivered over interval */ - s32 delivered_ce; /* number of packets delivered w/ CE marks*/ + s32 delivered_ce; /* packets delivered w/ CE mark over interval */ long interval_us; /* time for tp->delivered to incr "delivered" */ u32 snd_interval_us; /* snd interval for delivered packets */ u32 rcv_interval_us; /* rcv interval for delivered packets */ @@ -1214,7 +1244,9 @@ struct rate_sample { u32 last_end_seq; /* end_seq of most recently ACKed packet */ bool is_app_limited; /* is sample from packet with bubble in pipe? */ bool is_retrans; /* is sample from retransmission? */ + bool is_acking_tlp_retrans_seq; /* ACKed a TLP retransmit sequence? */ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + bool is_ece; /* did this ACK have ECN marked? */ }; struct tcp_congestion_ops { @@ -1238,8 +1270,11 @@ struct tcp_congestion_ops { /* hook for packet ack accounting (optional) */ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - /* override sysctl_tcp_min_tso_segs */ - u32 (*min_tso_segs)(struct sock *sk); + /* pick target number of segments per TSO/GSO skb (optional): */ + u32 (*tso_segs)(struct sock *sk, unsigned int mss_now); + + /* react to a specific lost skb (optional) */ + void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb); /* call when packets are delivered to update cwnd and pacing rate, * after all the ca_state processing. (optional) @@ -1305,6 +1340,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) } #endif +static inline bool tcp_ca_wants_ce_events(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN | + TCP_CONG_WANTS_CE_EVENTS); +} + static inline bool tcp_ca_needs_ecn(const struct sock *sk) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -1324,6 +1367,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) void tcp_set_ca_state(struct sock *sk, const u8 ca_state); /* From tcp_rate.c */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, struct rate_sample *rs); @@ -1336,6 +1380,21 @@ static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) return t1 > t2 || (t1 == t2 && after(seq1, seq2)); } +/* If a retransmit failed due to local qdisc congestion or other local issues, + * then we may have called tcp_set_skb_tso_segs() to increase the number of + * segments in the skb without increasing the tx.in_flight. In all other cases, + * the tx.in_flight should be at least as big as the pcount of the sk_buff. We + * do not have the state to know whether a retransmit failed due to local qdisc + * congestion or other local issues, so to avoid spurious warnings we consider + * that any skb marked lost may have suffered that fate. + */ +static inline bool tcp_skb_tx_in_flight_is_suspicious(u32 skb_pcount, + u32 skb_sacked_flags, + u32 tx_in_flight) +{ + return (skb_pcount > tx_in_flight) && !(skb_sacked_flags & TCPCB_LOST); +} + /* These functions determine how the current flow behaves in respect of SACK * handling. SACK is negotiated with the peer, and therefore it can vary * between different flows. @@ -2489,7 +2548,7 @@ struct tcp_plb_state { u8 consec_cong_rounds:5, /* consecutive congested rounds */ unused:3; u32 pause_until; /* jiffies32 when PLB can resume rerouting */ -}; +} __attribute__ ((__packed__)); static inline void tcp_plb_init(const struct sock *sk, struct tcp_plb_state *plb) diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h index 86bb2e8b17c9..9d9a3eb2ce9b 100644 --- a/include/uapi/linux/inet_diag.h +++ b/include/uapi/linux/inet_diag.h @@ -229,6 +229,29 @@ struct tcp_bbr_info { __u32 bbr_min_rtt; /* min-filtered RTT in uSec */ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */ + __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */ + __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */ + __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */ + __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */ + __u8 bbr_mode; /* current bbr_mode in state machine */ + __u8 bbr_phase; /* current state machine phase */ + __u8 unused1; /* alignment padding; not used yet */ + __u8 bbr_version; /* BBR algorithm version */ + __u32 bbr_inflight_lo; /* lower short-term data volume bound */ + __u32 bbr_inflight_hi; /* higher long-term data volume bound */ + __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */ +}; + +/* TCP BBR congestion control bbr_phase as reported in netlink/ss stats. */ +enum tcp_bbr_phase { + BBR_PHASE_INVALID = 0, + BBR_PHASE_STARTUP = 1, + BBR_PHASE_DRAIN = 2, + BBR_PHASE_PROBE_RTT = 3, + BBR_PHASE_PROBE_BW_UP = 4, + BBR_PHASE_PROBE_BW_DOWN = 5, + BBR_PHASE_PROBE_BW_CRUISE = 6, + BBR_PHASE_PROBE_BW_REFILL = 7, }; union tcp_cc_info { diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h index dab9493c791b..cce4975fdcfe 100644 --- a/include/uapi/linux/rtnetlink.h +++ b/include/uapi/linux/rtnetlink.h @@ -517,12 +517,14 @@ enum { #define RTAX_FEATURE_TIMESTAMP (1 << 2) /* unused */ #define RTAX_FEATURE_ALLFRAG (1 << 3) /* unused */ #define RTAX_FEATURE_TCP_USEC_TS (1 << 4) +#define RTAX_FEATURE_ECN_LOW (1 << 5) #define RTAX_FEATURE_MASK (RTAX_FEATURE_ECN | \ RTAX_FEATURE_SACK | \ RTAX_FEATURE_TIMESTAMP | \ RTAX_FEATURE_ALLFRAG | \ - RTAX_FEATURE_TCP_USEC_TS) + RTAX_FEATURE_TCP_USEC_TS | \ + RTAX_FEATURE_ECN_LOW) struct rta_session { __u8 proto; diff --git a/include/uapi/linux/tcp.h b/include/uapi/linux/tcp.h index dc8fdc80e16b..6b2003dbae81 100644 --- a/include/uapi/linux/tcp.h +++ b/include/uapi/linux/tcp.h @@ -184,6 +184,7 @@ enum tcp_fastopen_client_fail { #define TCPI_OPT_ECN_SEEN 16 /* we received at least one packet with ECT */ #define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */ #define TCPI_OPT_USEC_TS 64 /* usec timestamps */ +#define TCPI_OPT_ECN_LOW 128 /* Low-latency ECN enabled at conn init */ /* * Sender's congestion state indicating normal or abnormal situations diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig index 6d2c97f8e9ef..ddc116ef22cb 100644 --- a/net/ipv4/Kconfig +++ b/net/ipv4/Kconfig @@ -669,15 +669,18 @@ config TCP_CONG_BBR default n help - BBR (Bottleneck Bandwidth and RTT) TCP congestion control aims to - maximize network utilization and minimize queues. It builds an explicit - model of the bottleneck delivery rate and path round-trip propagation - delay. It tolerates packet loss and delay unrelated to congestion. It - can operate over LAN, WAN, cellular, wifi, or cable modem links. It can - coexist with flows that use loss-based congestion control, and can - operate with shallow buffers, deep buffers, bufferbloat, policers, or - AQM schemes that do not provide a delay signal. It requires the fq - ("Fair Queue") pacing packet scheduler. + BBR (Bottleneck Bandwidth and RTT) TCP congestion control is a + model-based congestion control algorithm that aims to maximize + network utilization, keep queues and retransmit rates low, and to be + able to coexist with Reno/CUBIC in common scenarios. It builds an + explicit model of the network path. It tolerates a targeted degree + of random packet loss and delay. It can operate over LAN, WAN, + cellular, wifi, or cable modem links, and can use shallow-threshold + ECN signals. It can coexist to some degree with flows that use + loss-based congestion control, and can operate with shallow buffers, + deep buffers, bufferbloat, policers, or AQM schemes that do not + provide a delay signal. It requires pacing, using either TCP internal + pacing or the fq ("Fair Queue") pacing packet scheduler. choice prompt "Default TCP congestion control" diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index e01492234b0b..27893b774e08 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -280,7 +280,7 @@ static void bpf_tcp_ca_pkts_acked(struct sock *sk, const struct ack_sample *samp { } -static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk) +static u32 bpf_tcp_ca_tso_segs(struct sock *sk, unsigned int mss_now) { return 0; } @@ -315,7 +315,7 @@ static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = { .cwnd_event = bpf_tcp_ca_cwnd_event, .in_ack_event = bpf_tcp_ca_in_ack_event, .pkts_acked = bpf_tcp_ca_pkts_acked, - .min_tso_segs = bpf_tcp_ca_min_tso_segs, + .tso_segs = bpf_tcp_ca_tso_segs, .cong_control = bpf_tcp_ca_cong_control, .undo_cwnd = bpf_tcp_ca_undo_cwnd, .sndbuf_expand = bpf_tcp_ca_sndbuf_expand, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 6edc441b3702..bf52c5744acf 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3411,6 +3411,7 @@ int tcp_disconnect(struct sock *sk, int flags) tp->rx_opt.dsack = 0; tp->rx_opt.num_sacks = 0; tp->rcv_ooopack = 0; + tp->fast_ack_mode = 0; /* Clean up fastopen related fields */ @@ -4158,6 +4159,8 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info) info->tcpi_options |= TCPI_OPT_ECN; if (tp->ecn_flags & TCP_ECN_SEEN) info->tcpi_options |= TCPI_OPT_ECN_SEEN; + if (tp->ecn_flags & TCP_ECN_LOW) + info->tcpi_options |= TCPI_OPT_ECN_LOW; if (tp->syn_data_acked) info->tcpi_options |= TCPI_OPT_SYN_DATA; if (tp->tcp_usec_ts) diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 760941e55153..066da5e5747c 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -1,18 +1,19 @@ -/* Bottleneck Bandwidth and RTT (BBR) congestion control +/* BBR (Bottleneck Bandwidth and RTT) congestion control * - * BBR congestion control computes the sending rate based on the delivery - * rate (throughput) estimated from ACKs. In a nutshell: + * BBR is a model-based congestion control algorithm that aims for low queues, + * low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model of the + * network path, it uses measurements of bandwidth and RTT, as well as (if they + * occur) packet loss and/or shallow-threshold ECN signals. Note that although + * it can use ECN or loss signals explicitly, it does not require either; it + * can bound its in-flight data based on its estimate of the BDP. * - * On each ACK, update our model of the network path: - * bottleneck_bandwidth = windowed_max(delivered / elapsed, 10 round trips) - * min_rtt = windowed_min(rtt, 10 seconds) - * pacing_rate = pacing_gain * bottleneck_bandwidth - * cwnd = max(cwnd_gain * bottleneck_bandwidth * min_rtt, 4) - * - * The core algorithm does not react directly to packet losses or delays, - * although BBR may adjust the size of next send per ACK when loss is - * observed, or adjust the sending rate if it estimates there is a - * traffic policer, in order to keep the drop rate reasonable. + * The model has both higher and lower bounds for the operating range: + * lo: bw_lo, inflight_lo: conservative short-term lower bound + * hi: bw_hi, inflight_hi: robust long-term upper bound + * The bandwidth-probing time scale is (a) extended dynamically based on + * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by + * an interactive wall-clock time-scale to be more scalable and responsive + * than Reno and CUBIC. * * Here is a state transition diagram for BBR: * @@ -65,6 +66,13 @@ #include #include +#include +#include "tcp_dctcp.h" + +#define BBR_VERSION 3 + +#define bbr_param(sk,name) (bbr_ ## name) + /* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps. * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32. @@ -85,36 +93,41 @@ enum bbr_mode { BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */ }; +/* How does the incoming ACK stream relate to our bandwidth probing? */ +enum bbr_ack_phase { + BBR_ACKS_INIT, /* not probing; not getting probe feedback */ + BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */ + BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */ + BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */ + BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */ +}; + /* BBR congestion control block */ struct bbr { u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */ u32 min_rtt_stamp; /* timestamp of min_rtt_us */ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */ - struct minmax bw; /* Max recent delivery rate in pkts/uS << 24 */ - u32 rtt_cnt; /* count of packet-timed rounds elapsed */ + u32 probe_rtt_min_us; /* min RTT in probe_rtt_win_ms win */ + u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */ u64 cycle_mstamp; /* time of this cycle phase start */ - u32 mode:3, /* current bbr_mode in state machine */ + u32 mode:2, /* current bbr_mode in state machine */ prev_ca_state:3, /* CA state on previous ACK */ - packet_conservation:1, /* use packet conservation? */ round_start:1, /* start of packet-timed tx->ack round? */ + ce_state:1, /* If most recent data has CE bit set */ + bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */ + try_fast_path:1, /* can we take fast path? */ idle_restart:1, /* restarting after idle? */ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ - unused:13, - lt_is_sampling:1, /* taking long-term ("LT") samples now? */ - lt_rtt_cnt:7, /* round trips in long-term interval */ - lt_use_bw:1; /* use lt_bw as our bw estimate? */ - u32 lt_bw; /* LT est delivery rate in pkts/uS << 24 */ - u32 lt_last_delivered; /* LT intvl start: tp->delivered */ - u32 lt_last_stamp; /* LT intvl start: tp->delivered_mstamp */ - u32 lt_last_lost; /* LT intvl start: tp->lost */ + init_cwnd:7, /* initial cwnd */ + unused_1:10; u32 pacing_gain:10, /* current gain for setting pacing rate */ cwnd_gain:10, /* current gain for setting cwnd */ full_bw_reached:1, /* reached full bw in Startup? */ full_bw_cnt:2, /* number of rounds without large bw gains */ - cycle_idx:3, /* current index in pacing_gain cycle array */ + cycle_idx:2, /* current index in pacing_gain cycle array */ has_seen_rtt:1, /* have we seen an RTT sample yet? */ - unused_b:5; + unused_2:6; u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ u32 full_bw; /* recent bw, to estimate if pipe is full */ @@ -124,19 +137,67 @@ struct bbr { u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ extra_acked_win_idx:1, /* current index in extra_acked array */ - unused_c:6; + /* BBR v3 state: */ + full_bw_now:1, /* recently reached full bw plateau? */ + startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */ + loss_in_cycle:1, /* packet loss in this cycle? */ + ecn_in_cycle:1, /* ECN in this cycle? */ + unused_3:1; + u32 loss_round_delivered; /* scb->tx.delivered ending loss round */ + u32 undo_bw_lo; /* bw_lo before latest losses */ + u32 undo_inflight_lo; /* inflight_lo before latest losses */ + u32 undo_inflight_hi; /* inflight_hi before latest losses */ + u32 bw_latest; /* max delivered bw in last round trip */ + u32 bw_lo; /* lower bound on sending bandwidth */ + u32 bw_hi[2]; /* max recent measured bw sample */ + u32 inflight_latest; /* max delivered data in last round trip */ + u32 inflight_lo; /* lower bound of inflight data range */ + u32 inflight_hi; /* upper bound of inflight data range */ + u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */ + u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */ + u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */ + u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */ + u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */ + ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */ + bw_probe_samples:1, /* rate samples reflect bw probing? */ + prev_probe_too_high:1, /* did last PROBE_UP go too high? */ + stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */ + rounds_since_probe:8, /* packet-timed rounds since probed bw */ + loss_round_start:1, /* loss_round_delivered round trip? */ + loss_in_round:1, /* loss marked in this round trip? */ + ecn_in_round:1, /* ECN marked in this round trip? */ + ack_phase:3, /* bbr_ack_phase: meaning of ACKs */ + loss_events_in_round:4,/* losses in STARTUP round */ + initialized:1; /* has bbr_init() been called? */ + u32 alpha_last_delivered; /* tp->delivered at alpha update */ + u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */ + + u8 unused_4; /* to preserve alignment */ + struct tcp_plb_state plb; }; -#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ +struct bbr_context { + u32 sample_bw; +}; -/* Window length of bw filter (in rounds): */ -static const int bbr_bw_rtts = CYCLE_LEN + 2; /* Window length of min_rtt filter (in sec): */ static const u32 bbr_min_rtt_win_sec = 10; /* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode: */ static const u32 bbr_probe_rtt_mode_ms = 200; -/* Skip TSO below the following bandwidth (bits/sec): */ -static const int bbr_min_tso_rate = 1200000; +/* Window length of probe_rtt_min_us filter (in ms), and consequently the + * typical interval between PROBE_RTT mode entries. The default is 5000ms. + * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC + */ +static const u32 bbr_probe_rtt_win_ms = 5000; +/* Proportion of cwnd to estimated BDP in PROBE_RTT, in units of BBR_UNIT: */ +static const u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2; + +/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting + * in bigger TSO bursts. We cut the RTT-based allowance in half + * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance + * is below 1500 bytes after 6 * ~500 usec = 3ms. + */ +static const u32 bbr_tso_rtt_shift = 9; /* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck. * In order to help drive the network toward lower queues and low latency while @@ -146,13 +207,15 @@ static const int bbr_min_tso_rate = 1200000; */ static const int bbr_pacing_margin_percent = 1; -/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain +/* We use a startup_pacing_gain of 4*ln(2) because it's the smallest value * that will allow a smoothly increasing pacing rate that will double each RTT * and send the same number of packets per RTT that an un-paced, slow-starting * Reno or CUBIC flow would: */ -static const int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; -/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain +static const int bbr_startup_pacing_gain = BBR_UNIT * 277 / 100 + 1; +/* The gain for deriving startup cwnd: */ +static const int bbr_startup_cwnd_gain = BBR_UNIT * 2; +/* The pacing gain in BBR_DRAIN is calculated to typically drain * the queue created in BBR_STARTUP in a single round: */ static const int bbr_drain_gain = BBR_UNIT * 1000 / 2885; @@ -160,13 +223,17 @@ static const int bbr_drain_gain = BBR_UNIT * 1000 / 2885; static const int bbr_cwnd_gain = BBR_UNIT * 2; /* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw: */ static const int bbr_pacing_gain[] = { - BBR_UNIT * 5 / 4, /* probe for more available bw */ - BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ - BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */ - BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */ + BBR_UNIT * 5 / 4, /* UP: probe for more available bw */ + BBR_UNIT * 91 / 100, /* DOWN: drain queue and/or yield bw */ + BBR_UNIT, /* CRUISE: try to use pipe w/ some headroom */ + BBR_UNIT, /* REFILL: refill pipe to estimated 100% */ +}; +enum bbr_pacing_gain_phase { + BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */ + BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */ + BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */ + BBR_BW_PROBE_REFILL = 3, /* refill the pipe again to 100% */ }; -/* Randomize the starting gain cycling phase over N phases: */ -static const u32 bbr_cycle_rand = 7; /* Try to keep at least this many packets in flight, if things go smoothly. For * smooth functioning, a sliding window protocol ACKing every other packet @@ -174,24 +241,12 @@ static const u32 bbr_cycle_rand = 7; */ static const u32 bbr_cwnd_min_target = 4; -/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */ +/* To estimate if BBR_STARTUP or BBR_BW_PROBE_UP has filled pipe... */ /* If bw has increased significantly (1.25x), there may be more bw available: */ static const u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4; /* But after 3 rounds w/o significant bw growth, estimate pipe is full: */ static const u32 bbr_full_bw_cnt = 3; -/* "long-term" ("LT") bandwidth estimator parameters... */ -/* The minimum number of rounds in an LT bw sampling interval: */ -static const u32 bbr_lt_intvl_min_rtts = 4; -/* If lost/delivered ratio > 20%, interval is "lossy" and we may be policed: */ -static const u32 bbr_lt_loss_thresh = 50; -/* If 2 intervals have a bw ratio <= 1/8, their bw is "consistent": */ -static const u32 bbr_lt_bw_ratio = BBR_UNIT / 8; -/* If 2 intervals have a bw diff <= 4 Kbit/sec their bw is "consistent": */ -static const u32 bbr_lt_bw_diff = 4000 / 8; -/* If we estimate we're policed, use lt_bw for this many round trips: */ -static const u32 bbr_lt_bw_max_rtts = 48; - /* Gain factor for adding extra_acked to target cwnd: */ static const int bbr_extra_acked_gain = BBR_UNIT; /* Window length of extra_acked window. */ @@ -201,8 +256,122 @@ static const u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; /* Time period for clamping cwnd increment due to ack aggregation */ static const u32 bbr_extra_acked_max_us = 100 * 1000; +/* Flags to control BBR ECN-related behavior... */ + +/* Ensure ACKs only ACK packets with consistent ECN CE status? */ +static const bool bbr_precise_ece_ack = true; + +/* Max RTT (in usec) at which to use sender-side ECN logic. + * Disabled when 0 (ECN allowed at any RTT). + */ +static const u32 bbr_ecn_max_rtt_us = 5000; + +/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE. + * No loss response when 0. + */ +static const u32 bbr_beta = BBR_UNIT * 30 / 100; + +/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE (1/16 = 6.25%) */ +static const u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; + +/* The initial value for ecn_alpha; 1.0 allows a flow to respond quickly + * to congestion if the bottleneck is congested when the flow starts up. + */ +static const u32 bbr_ecn_alpha_init = BBR_UNIT; + +/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE. + * No ECN based bounding when 0. + */ +static const u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */ + +/* Estimate bw probing has gone too far if CE ratio exceeds this threshold. + * Scaled by BBR_SCALE. Disabled when 0. + */ +static const u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */ + +/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN + * clears then make the first round's increment to inflight_hi the following + * fraction of inflight_hi. + */ +static const u32 bbr_ecn_reprobe_gain = BBR_UNIT * 1 / 2; + +/* Estimate bw probing has gone too far if loss rate exceeds this level. */ +static const u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */ + +/* Slow down for a packet loss recovered by TLP? */ +static const bool bbr_loss_probe_recovery = true; + +/* Exit STARTUP if number of loss marking events in a Recovery round is >= N, + * and loss rate is higher than bbr_loss_thresh. + * Disabled if 0. + */ +static const u32 bbr_full_loss_cnt = 6; + +/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh + * meets this count. + */ +static const u32 bbr_full_ecn_cnt = 2; + +/* Fraction of unutilized headroom to try to leave in path upon high loss. */ +static const u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100; + +/* How much do we increase cwnd_gain when probing for bandwidth in + * BBR_BW_PROBE_UP? This specifies the increment in units of + * BBR_UNIT/4. The default is 1, meaning 0.25. + * The min value is 0 (meaning 0.0); max is 3 (meaning 0.75). + */ +static const u32 bbr_bw_probe_cwnd_gain = 1; + +/* Max number of packet-timed rounds to wait before probing for bandwidth. If + * we want to tolerate 1% random loss per round, and not have this cut our + * inflight too much, we must probe for bw periodically on roughly this scale. + * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance. + * We aim to be fair with Reno/CUBIC up to a BDP of at least: + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + */ +static const u32 bbr_bw_probe_max_rounds = 63; + +/* Max amount of randomness to inject in round counting for Reno-coexistence. + */ +static const u32 bbr_bw_probe_rand_rounds = 2; + +/* Use BBR-native probe time scale starting at this many usec. + * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least: + * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs + */ +static const u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */ + +/* Use BBR-native probes spread over this many usec: */ +static const u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */ + +/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */ +static const bool bbr_fast_path = true; + +/* Use fast ack mode? */ +static const bool bbr_fast_ack_mode = true; + +static u32 bbr_max_bw(const struct sock *sk); +static u32 bbr_bw(const struct sock *sk); +static void bbr_exit_probe_rtt(struct sock *sk); +static void bbr_reset_congestion_signals(struct sock *sk); +static void bbr_run_loss_probe_recovery(struct sock *sk); + static void bbr_check_probe_rtt_done(struct sock *sk); +/* This connection can use ECN if both endpoints have signaled ECN support in + * the handshake and the per-route settings indicated this is a + * shallow-threshold ECN environment, meaning both: + * (a) ECN CE marks indicate low-latency/shallow-threshold congestion, and + * (b) TCP endpoints provide precise ACKs that only ACK data segments + * with consistent ECN CE status + */ +static bool bbr_can_use_ecn(const struct sock *sk) +{ + const struct tcp_sock *tp = tcp_sk(sk); + + return (tcp_ecn_mode_any(tp)) && (tp->ecn_flags & TCP_ECN_LOW); +} + /* Do we estimate that STARTUP filled the pipe? */ static bool bbr_full_bw_reached(const struct sock *sk) { @@ -214,17 +383,17 @@ static bool bbr_full_bw_reached(const struct sock *sk) /* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */ static u32 bbr_max_bw(const struct sock *sk) { - struct bbr *bbr = inet_csk_ca(sk); + const struct bbr *bbr = inet_csk_ca(sk); - return minmax_get(&bbr->bw); + return max(bbr->bw_hi[0], bbr->bw_hi[1]); } /* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */ static u32 bbr_bw(const struct sock *sk) { - struct bbr *bbr = inet_csk_ca(sk); + const struct bbr *bbr = inet_csk_ca(sk); - return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk); + return min(bbr_max_bw(sk), bbr->bw_lo); } /* Return maximum extra acked in past k-2k round trips, @@ -241,15 +410,23 @@ static u16 bbr_extra_acked(const struct sock *sk) * The order here is chosen carefully to avoid overflow of u64. This should * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. */ -static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain) +static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain, + int margin) { unsigned int mss = tcp_sk(sk)->mss_cache; rate *= mss; rate *= gain; rate >>= BBR_SCALE; - rate *= USEC_PER_SEC / 100 * (100 - bbr_pacing_margin_percent); - return rate >> BW_SCALE; + rate *= USEC_PER_SEC / 100 * (100 - margin); + rate >>= BW_SCALE; + rate = max(rate, 1ULL); + return rate; +} + +static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate) +{ + return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0); } /* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */ @@ -257,12 +434,13 @@ static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain) { u64 rate = bw; - rate = bbr_rate_bytes_per_sec(sk, rate, gain); + rate = bbr_rate_bytes_per_sec(sk, rate, gain, + bbr_pacing_margin_percent); rate = min_t(u64, rate, READ_ONCE(sk->sk_max_pacing_rate)); return rate; } -/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */ +/* Initialize pacing rate to: startup_pacing_gain * init_cwnd / RTT. */ static void bbr_init_pacing_rate_from_rtt(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); @@ -279,7 +457,8 @@ static void bbr_init_pacing_rate_from_rtt(struct sock *sk) bw = (u64)tcp_snd_cwnd(tp) * BW_UNIT; do_div(bw, rtt_us); WRITE_ONCE(sk->sk_pacing_rate, - bbr_bw_to_pacing_rate(sk, bw, bbr_high_gain)); + bbr_bw_to_pacing_rate(sk, bw, + bbr_param(sk, startup_pacing_gain))); } /* Pace using current bw estimate and a gain factor. */ @@ -295,26 +474,48 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain) WRITE_ONCE(sk->sk_pacing_rate, rate); } -/* override sysctl_tcp_min_tso_segs */ -__bpf_kfunc static u32 bbr_min_tso_segs(struct sock *sk) +/* Return the number of segments BBR would like in a TSO/GSO skb, given a + * particular max gso size as a constraint. TODO: make this simpler and more + * consistent by switching bbr to just call tcp_tso_autosize(). + */ +static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now, + u32 gso_max_size) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 segs, r; + u64 bytes; + + /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */ + bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift); + + /* Budget a TSO/GSO burst size allowance based on min_rtt. For every + * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst. + * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K) + */ + if (bbr_param(sk, tso_rtt_shift)) { + r = bbr->min_rtt_us >> bbr_param(sk, tso_rtt_shift); + if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */ + bytes += GSO_LEGACY_MAX_SIZE >> r; + } + + bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + return segs; +} + +/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */ +__bpf_kfunc static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now) { - return READ_ONCE(sk->sk_pacing_rate) < (bbr_min_tso_rate >> 3) ? 1 : 2; + return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size); } +/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */ static u32 bbr_tso_segs_goal(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); - u32 segs, bytes; - - /* Sort of tcp_tso_autosize() but ignoring - * driver provided sk_gso_max_size. - */ - bytes = min_t(unsigned long, - READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift), - GSO_LEGACY_MAX_SIZE - 1 - MAX_TCP_HEADER); - segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); - return min(segs, 0x7FU); + return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_LEGACY_MAX_SIZE); } /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ @@ -334,7 +535,9 @@ __bpf_kfunc static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - if (event == CA_EVENT_TX_START && tp->app_limited) { + if (event == CA_EVENT_TX_START) { + if (!tp->app_limited) + return; bbr->idle_restart = 1; bbr->ack_epoch_mstamp = tp->tcp_mstamp; bbr->ack_epoch_acked = 0; @@ -345,6 +548,16 @@ __bpf_kfunc static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event) bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); else if (bbr->mode == BBR_PROBE_RTT) bbr_check_probe_rtt_done(sk); + } else if ((event == CA_EVENT_ECN_IS_CE || + event == CA_EVENT_ECN_NO_CE) && + bbr_can_use_ecn(sk) && + bbr_param(sk, precise_ece_ack)) { + u32 state = bbr->ce_state; + dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state); + bbr->ce_state = state; + } else if (event == CA_EVENT_TLP_RECOVERY && + bbr_param(sk, loss_probe_recovery)) { + bbr_run_loss_probe_recovery(sk); } } @@ -367,10 +580,10 @@ static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) * default. This should only happen when the connection is not using TCP * timestamps and has retransmitted all of the SYN/SYNACK/data packets * ACKed so far. In this case, an RTO can cut cwnd to 1, in which - * case we need to slow-start up toward something safe: TCP_INIT_CWND. + * case we need to slow-start up toward something safe: initial cwnd. */ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ - return TCP_INIT_CWND; /* be safe: cap at default initial cwnd*/ + return bbr->init_cwnd; /* be safe: cap at initial cwnd */ w = (u64)bw * bbr->min_rtt_us; @@ -387,23 +600,23 @@ static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) * - one skb in sending host Qdisc, * - one skb in sending host TSO/GSO engine * - one skb being received by receiver host LRO/GRO/delayed-ACK engine - * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because - * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, + * Don't worry, at low rates this won't bloat cwnd because + * in such cases tso_segs_goal is small. The minimum cwnd is 4 packets, * which allows 2 outstanding 2-packet sequences, to try to keep pipe * full even with ACK-every-other-packet delayed ACKs. */ static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) { struct bbr *bbr = inet_csk_ca(sk); + u32 tso_segs_goal; - /* Allow enough full-sized skbs in flight to utilize end systems. */ - cwnd += 3 * bbr_tso_segs_goal(sk); - - /* Reduce delayed ACKs by rounding up cwnd to the next even number. */ - cwnd = (cwnd + 1) & ~1U; + tso_segs_goal = 3 * bbr_tso_segs_goal(sk); + /* Allow enough full-sized skbs in flight to utilize end systems. */ + cwnd = max_t(u32, cwnd, tso_segs_goal); + cwnd = max_t(u32, cwnd, bbr_param(sk, cwnd_min_target)); /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ - if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0) + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) cwnd += 2; return cwnd; @@ -458,10 +671,10 @@ static u32 bbr_ack_aggregation_cwnd(struct sock *sk) { u32 max_aggr_cwnd, aggr_cwnd = 0; - if (bbr_extra_acked_gain && bbr_full_bw_reached(sk)) { + if (bbr_param(sk, extra_acked_gain)) { max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us) / BW_UNIT; - aggr_cwnd = (bbr_extra_acked_gain * bbr_extra_acked(sk)) + aggr_cwnd = (bbr_param(sk, extra_acked_gain) * bbr_extra_acked(sk)) >> BBR_SCALE; aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd); } @@ -469,66 +682,27 @@ static u32 bbr_ack_aggregation_cwnd(struct sock *sk) return aggr_cwnd; } -/* An optimization in BBR to reduce losses: On the first round of recovery, we - * follow the packet conservation principle: send P packets per P packets acked. - * After that, we slow-start and send at most 2*P packets per P packets acked. - * After recovery finishes, or upon undo, we restore the cwnd we had when - * recovery started (capped by the target cwnd based on estimated BDP). - * - * TODO(ycheng/ncardwell): implement a rate-based approach. - */ -static bool bbr_set_cwnd_to_recover_or_restore( - struct sock *sk, const struct rate_sample *rs, u32 acked, u32 *new_cwnd) +/* Returns the cwnd for PROBE_RTT mode. */ +static u32 bbr_probe_rtt_cwnd(struct sock *sk) { - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state; - u32 cwnd = tcp_snd_cwnd(tp); - - /* An ACK for P pkts should release at most 2*P packets. We do this - * in two steps. First, here we deduct the number of lost packets. - * Then, in bbr_set_cwnd() we slow start up toward the target cwnd. - */ - if (rs->losses > 0) - cwnd = max_t(s32, cwnd - rs->losses, 1); - - if (state == TCP_CA_Recovery && prev_state != TCP_CA_Recovery) { - /* Starting 1st round of Recovery, so do packet conservation. */ - bbr->packet_conservation = 1; - bbr->next_rtt_delivered = tp->delivered; /* start round now */ - /* Cut unused cwnd from app behavior, TSQ, or TSO deferral: */ - cwnd = tcp_packets_in_flight(tp) + acked; - } else if (prev_state >= TCP_CA_Recovery && state < TCP_CA_Recovery) { - /* Exiting loss recovery; restore cwnd saved before recovery. */ - cwnd = max(cwnd, bbr->prior_cwnd); - bbr->packet_conservation = 0; - } - bbr->prev_ca_state = state; - - if (bbr->packet_conservation) { - *new_cwnd = max(cwnd, tcp_packets_in_flight(tp) + acked); - return true; /* yes, using packet conservation */ - } - *new_cwnd = cwnd; - return false; + return max_t(u32, bbr_param(sk, cwnd_min_target), + bbr_bdp(sk, bbr_bw(sk), bbr_param(sk, probe_rtt_cwnd_gain))); } /* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss * has drawn us down below target), or snap down to target if we're above it. */ static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, - u32 acked, u32 bw, int gain) + u32 acked, u32 bw, int gain, u32 cwnd, + struct bbr_context *ctx) { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u32 cwnd = tcp_snd_cwnd(tp), target_cwnd = 0; + u32 target_cwnd = 0; if (!acked) goto done; /* no packet fully ACKed; just apply caps */ - if (bbr_set_cwnd_to_recover_or_restore(sk, rs, acked, &cwnd)) - goto done; - target_cwnd = bbr_bdp(sk, bw, gain); /* Increment the cwnd to account for excess ACKed data that seems @@ -537,74 +711,26 @@ static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs, target_cwnd += bbr_ack_aggregation_cwnd(sk); target_cwnd = bbr_quantization_budget(sk, target_cwnd); - /* If we're below target cwnd, slow start cwnd toward target cwnd. */ - if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ - cwnd = min(cwnd + acked, target_cwnd); - else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND) - cwnd = cwnd + acked; - cwnd = max(cwnd, bbr_cwnd_min_target); + /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */ + bbr->try_fast_path = 0; + if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */ + cwnd += acked; + if (cwnd >= target_cwnd) { + cwnd = target_cwnd; + bbr->try_fast_path = 1; + } + } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) { + cwnd += acked; + } else { + bbr->try_fast_path = 1; + } + cwnd = max_t(u32, cwnd, bbr_param(sk, cwnd_min_target)); done: - tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); /* apply global cap */ + tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp)); /* global cap */ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ - tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), bbr_cwnd_min_target)); -} - -/* End cycle phase if it's time and/or we hit the phase's in-flight target. */ -static bool bbr_is_next_cycle_phase(struct sock *sk, - const struct rate_sample *rs) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - bool is_full_length = - tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp) > - bbr->min_rtt_us; - u32 inflight, bw; - - /* The pacing_gain of 1.0 paces at the estimated bw to try to fully - * use the pipe without increasing the queue. - */ - if (bbr->pacing_gain == BBR_UNIT) - return is_full_length; /* just use wall clock time */ - - inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); - bw = bbr_max_bw(sk); - - /* A pacing_gain > 1.0 probes for bw by trying to raise inflight to at - * least pacing_gain*BDP; this may take more than min_rtt if min_rtt is - * small (e.g. on a LAN). We do not persist if packets are lost, since - * a path with small buffers may not hold that much. - */ - if (bbr->pacing_gain > BBR_UNIT) - return is_full_length && - (rs->losses || /* perhaps pacing_gain*BDP won't fit */ - inflight >= bbr_inflight(sk, bw, bbr->pacing_gain)); - - /* A pacing_gain < 1.0 tries to drain extra queue we added if bw - * probing didn't find more bw. If inflight falls to match BDP then we - * estimate queue is drained; persisting would underutilize the pipe. - */ - return is_full_length || - inflight <= bbr_inflight(sk, bw, BBR_UNIT); -} - -static void bbr_advance_cycle_phase(struct sock *sk) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - - bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1); - bbr->cycle_mstamp = tp->delivered_mstamp; -} - -/* Gain cycling: cycle pacing gain to converge to fair share of available bw. */ -static void bbr_update_cycle_phase(struct sock *sk, - const struct rate_sample *rs) -{ - struct bbr *bbr = inet_csk_ca(sk); - - if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs)) - bbr_advance_cycle_phase(sk); + tcp_snd_cwnd_set(tp, min_t(u32, tcp_snd_cwnd(tp), + bbr_probe_rtt_cwnd(sk))); } static void bbr_reset_startup_mode(struct sock *sk) @@ -614,191 +740,49 @@ static void bbr_reset_startup_mode(struct sock *sk) bbr->mode = BBR_STARTUP; } -static void bbr_reset_probe_bw_mode(struct sock *sk) -{ - struct bbr *bbr = inet_csk_ca(sk); - - bbr->mode = BBR_PROBE_BW; - bbr->cycle_idx = CYCLE_LEN - 1 - get_random_u32_below(bbr_cycle_rand); - bbr_advance_cycle_phase(sk); /* flip to next phase of gain cycle */ -} - -static void bbr_reset_mode(struct sock *sk) -{ - if (!bbr_full_bw_reached(sk)) - bbr_reset_startup_mode(sk); - else - bbr_reset_probe_bw_mode(sk); -} - -/* Start a new long-term sampling interval. */ -static void bbr_reset_lt_bw_sampling_interval(struct sock *sk) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - - bbr->lt_last_stamp = div_u64(tp->delivered_mstamp, USEC_PER_MSEC); - bbr->lt_last_delivered = tp->delivered; - bbr->lt_last_lost = tp->lost; - bbr->lt_rtt_cnt = 0; -} - -/* Completely reset long-term bandwidth sampling. */ -static void bbr_reset_lt_bw_sampling(struct sock *sk) -{ - struct bbr *bbr = inet_csk_ca(sk); - - bbr->lt_bw = 0; - bbr->lt_use_bw = 0; - bbr->lt_is_sampling = false; - bbr_reset_lt_bw_sampling_interval(sk); -} - -/* Long-term bw sampling interval is done. Estimate whether we're policed. */ -static void bbr_lt_bw_interval_done(struct sock *sk, u32 bw) -{ - struct bbr *bbr = inet_csk_ca(sk); - u32 diff; - - if (bbr->lt_bw) { /* do we have bw from a previous interval? */ - /* Is new bw close to the lt_bw from the previous interval? */ - diff = abs(bw - bbr->lt_bw); - if ((diff * BBR_UNIT <= bbr_lt_bw_ratio * bbr->lt_bw) || - (bbr_rate_bytes_per_sec(sk, diff, BBR_UNIT) <= - bbr_lt_bw_diff)) { - /* All criteria are met; estimate we're policed. */ - bbr->lt_bw = (bw + bbr->lt_bw) >> 1; /* avg 2 intvls */ - bbr->lt_use_bw = 1; - bbr->pacing_gain = BBR_UNIT; /* try to avoid drops */ - bbr->lt_rtt_cnt = 0; - return; - } - } - bbr->lt_bw = bw; - bbr_reset_lt_bw_sampling_interval(sk); -} - -/* Token-bucket traffic policers are common (see "An Internet-Wide Analysis of - * Traffic Policing", SIGCOMM 2016). BBR detects token-bucket policers and - * explicitly models their policed rate, to reduce unnecessary losses. We - * estimate that we're policed if we see 2 consecutive sampling intervals with - * consistent throughput and high packet loss. If we think we're being policed, - * set lt_bw to the "long-term" average delivery rate from those 2 intervals. +/* See if we have reached next round trip. Upon start of the new round, + * returns packets delivered since previous round start plus this ACK. */ -static void bbr_lt_bw_sampling(struct sock *sk, const struct rate_sample *rs) -{ - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); - u32 lost, delivered; - u64 bw; - u32 t; - - if (bbr->lt_use_bw) { /* already using long-term rate, lt_bw? */ - if (bbr->mode == BBR_PROBE_BW && bbr->round_start && - ++bbr->lt_rtt_cnt >= bbr_lt_bw_max_rtts) { - bbr_reset_lt_bw_sampling(sk); /* stop using lt_bw */ - bbr_reset_probe_bw_mode(sk); /* restart gain cycling */ - } - return; - } - - /* Wait for the first loss before sampling, to let the policer exhaust - * its tokens and estimate the steady-state rate allowed by the policer. - * Starting samples earlier includes bursts that over-estimate the bw. - */ - if (!bbr->lt_is_sampling) { - if (!rs->losses) - return; - bbr_reset_lt_bw_sampling_interval(sk); - bbr->lt_is_sampling = true; - } - - /* To avoid underestimates, reset sampling if we run out of data. */ - if (rs->is_app_limited) { - bbr_reset_lt_bw_sampling(sk); - return; - } - - if (bbr->round_start) - bbr->lt_rtt_cnt++; /* count round trips in this interval */ - if (bbr->lt_rtt_cnt < bbr_lt_intvl_min_rtts) - return; /* sampling interval needs to be longer */ - if (bbr->lt_rtt_cnt > 4 * bbr_lt_intvl_min_rtts) { - bbr_reset_lt_bw_sampling(sk); /* interval is too long */ - return; - } - - /* End sampling interval when a packet is lost, so we estimate the - * policer tokens were exhausted. Stopping the sampling before the - * tokens are exhausted under-estimates the policed rate. - */ - if (!rs->losses) - return; - - /* Calculate packets lost and delivered in sampling interval. */ - lost = tp->lost - bbr->lt_last_lost; - delivered = tp->delivered - bbr->lt_last_delivered; - /* Is loss rate (lost/delivered) >= lt_loss_thresh? If not, wait. */ - if (!delivered || (lost << BBR_SCALE) < bbr_lt_loss_thresh * delivered) - return; - - /* Find average delivery rate in this sampling interval. */ - t = div_u64(tp->delivered_mstamp, USEC_PER_MSEC) - bbr->lt_last_stamp; - if ((s32)t < 1) - return; /* interval is less than one ms, so wait */ - /* Check if can multiply without overflow */ - if (t >= ~0U / USEC_PER_MSEC) { - bbr_reset_lt_bw_sampling(sk); /* interval too long; reset */ - return; - } - t *= USEC_PER_MSEC; - bw = (u64)delivered * BW_UNIT; - do_div(bw, t); - bbr_lt_bw_interval_done(sk, bw); -} - -/* Estimate the bandwidth based on how fast packets are delivered */ -static void bbr_update_bw(struct sock *sk, const struct rate_sample *rs) +static u32 bbr_update_round_start(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u64 bw; + u32 round_delivered = 0; bbr->round_start = 0; - if (rs->delivered < 0 || rs->interval_us <= 0) - return; /* Not a valid observation */ /* See if we've reached the next RTT */ - if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) { + if (rs->interval_us > 0 && + !before(rs->prior_delivered, bbr->next_rtt_delivered)) { + round_delivered = tp->delivered - bbr->next_rtt_delivered; bbr->next_rtt_delivered = tp->delivered; - bbr->rtt_cnt++; bbr->round_start = 1; - bbr->packet_conservation = 0; } + return round_delivered; +} - bbr_lt_bw_sampling(sk, rs); +/* Calculate the bandwidth based on how fast packets are delivered */ +static void bbr_calculate_bw_sample(struct sock *sk, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + u64 bw = 0; /* Divide delivered by the interval to find a (lower bound) bottleneck * bandwidth sample. Delivered is in packets and interval_us in uS and * ratio will be <<1 for most connections. So delivered is first scaled. + * Round up to allow growth at low rates, even with integer division. */ - bw = div64_long((u64)rs->delivered * BW_UNIT, rs->interval_us); - - /* If this sample is application-limited, it is likely to have a very - * low delivered count that represents application behavior rather than - * the available network rate. Such a sample could drag down estimated - * bw, causing needless slow-down. Thus, to continue to send at the - * last measured network rate, we filter out app-limited samples unless - * they describe the path bw at least as well as our bw model. - * - * So the goal during app-limited phase is to proceed with the best - * network rate no matter how long. We automatically leave this - * phase when app writes faster than the network can deliver :) - */ - if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) { - /* Incorporate new sample into our max bw filter. */ - minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw); + if (rs->interval_us > 0) { + if (WARN_ONCE(rs->delivered < 0, + "negative delivered: %d interval_us: %ld\n", + rs->delivered, rs->interval_us)) + return; + + bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us); } + + ctx->sample_bw = bw; } /* Estimates the windowed max degree of ack aggregation. @@ -812,7 +796,7 @@ static void bbr_update_bw(struct sock *sk, const struct rate_sample *rs) * * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). * Max filter is an approximate sliding window of 5-10 (packet timed) round - * trips. + * trips for non-startup phase, and 1-2 round trips for startup. */ static void bbr_update_ack_aggregation(struct sock *sk, const struct rate_sample *rs) @@ -820,15 +804,19 @@ static void bbr_update_ack_aggregation(struct sock *sk, u32 epoch_us, expected_acked, extra_acked; struct bbr *bbr = inet_csk_ca(sk); struct tcp_sock *tp = tcp_sk(sk); + u32 extra_acked_win_rtts_thresh = bbr_param(sk, extra_acked_win_rtts); - if (!bbr_extra_acked_gain || rs->acked_sacked <= 0 || + if (!bbr_param(sk, extra_acked_gain) || rs->acked_sacked <= 0 || rs->delivered < 0 || rs->interval_us <= 0) return; if (bbr->round_start) { bbr->extra_acked_win_rtts = min(0x1F, bbr->extra_acked_win_rtts + 1); - if (bbr->extra_acked_win_rtts >= bbr_extra_acked_win_rtts) { + if (!bbr_full_bw_reached(sk)) + extra_acked_win_rtts_thresh = 1; + if (bbr->extra_acked_win_rtts >= + extra_acked_win_rtts_thresh) { bbr->extra_acked_win_rtts = 0; bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? 0 : 1; @@ -862,49 +850,6 @@ static void bbr_update_ack_aggregation(struct sock *sk, bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; } -/* Estimate when the pipe is full, using the change in delivery rate: BBR - * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by - * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited - * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the - * higher rwin, 3: we get higher delivery rate samples. Or transient - * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar - * design goal, but uses delay and inter-ACK spacing instead of bandwidth. - */ -static void bbr_check_full_bw_reached(struct sock *sk, - const struct rate_sample *rs) -{ - struct bbr *bbr = inet_csk_ca(sk); - u32 bw_thresh; - - if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) - return; - - bw_thresh = (u64)bbr->full_bw * bbr_full_bw_thresh >> BBR_SCALE; - if (bbr_max_bw(sk) >= bw_thresh) { - bbr->full_bw = bbr_max_bw(sk); - bbr->full_bw_cnt = 0; - return; - } - ++bbr->full_bw_cnt; - bbr->full_bw_reached = bbr->full_bw_cnt >= bbr_full_bw_cnt; -} - -/* If pipe is probably full, drain the queue and then enter steady-state. */ -static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs) -{ - struct bbr *bbr = inet_csk_ca(sk); - - if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { - bbr->mode = BBR_DRAIN; /* drain queue we created */ - tcp_sk(sk)->snd_ssthresh = - bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); - } /* fall through to check if in-flight is already small: */ - if (bbr->mode == BBR_DRAIN && - bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= - bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) - bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ -} - static void bbr_check_probe_rtt_done(struct sock *sk) { struct tcp_sock *tp = tcp_sk(sk); @@ -914,9 +859,9 @@ static void bbr_check_probe_rtt_done(struct sock *sk) after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) return; - bbr->min_rtt_stamp = tcp_jiffies32; /* wait a while until PROBE_RTT */ + bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */ tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); - bbr_reset_mode(sk); + bbr_exit_probe_rtt(sk); } /* The goal of PROBE_RTT mode is to have BBR flows cooperatively and @@ -942,23 +887,35 @@ static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) { struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - bool filter_expired; + bool probe_rtt_expired, min_rtt_expired; + u32 expire; - /* Track min RTT seen in the min_rtt_win_sec filter window: */ - filter_expired = after(tcp_jiffies32, - bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ); + /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */ + expire = bbr->probe_rtt_min_stamp + + msecs_to_jiffies(bbr_param(sk, probe_rtt_win_ms)); + probe_rtt_expired = after(tcp_jiffies32, expire); if (rs->rtt_us >= 0 && - (rs->rtt_us < bbr->min_rtt_us || - (filter_expired && !rs->is_ack_delayed))) { - bbr->min_rtt_us = rs->rtt_us; - bbr->min_rtt_stamp = tcp_jiffies32; + (rs->rtt_us < bbr->probe_rtt_min_us || + (probe_rtt_expired && !rs->is_ack_delayed))) { + bbr->probe_rtt_min_us = rs->rtt_us; + bbr->probe_rtt_min_stamp = tcp_jiffies32; + } + /* Track min RTT seen in the min_rtt_win_sec filter window: */ + expire = bbr->min_rtt_stamp + bbr_param(sk, min_rtt_win_sec) * HZ; + min_rtt_expired = after(tcp_jiffies32, expire); + if (bbr->probe_rtt_min_us <= bbr->min_rtt_us || + min_rtt_expired) { + bbr->min_rtt_us = bbr->probe_rtt_min_us; + bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp; } - if (bbr_probe_rtt_mode_ms > 0 && filter_expired && + if (bbr_param(sk, probe_rtt_mode_ms) > 0 && probe_rtt_expired && !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ bbr_save_cwnd(sk); /* note cwnd so we can restore it */ bbr->probe_rtt_done_stamp = 0; + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; } if (bbr->mode == BBR_PROBE_RTT) { @@ -967,9 +924,9 @@ static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs) (tp->delivered + tcp_packets_in_flight(tp)) ? : 1; /* Maintain min packets in flight for max(200 ms, 1 round). */ if (!bbr->probe_rtt_done_stamp && - tcp_packets_in_flight(tp) <= bbr_cwnd_min_target) { + tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) { bbr->probe_rtt_done_stamp = tcp_jiffies32 + - msecs_to_jiffies(bbr_probe_rtt_mode_ms); + msecs_to_jiffies(bbr_param(sk, probe_rtt_mode_ms)); bbr->probe_rtt_round_done = 0; bbr->next_rtt_delivered = tp->delivered; } else if (bbr->probe_rtt_done_stamp) { @@ -990,18 +947,20 @@ static void bbr_update_gains(struct sock *sk) switch (bbr->mode) { case BBR_STARTUP: - bbr->pacing_gain = bbr_high_gain; - bbr->cwnd_gain = bbr_high_gain; + bbr->pacing_gain = bbr_param(sk, startup_pacing_gain); + bbr->cwnd_gain = bbr_param(sk, startup_cwnd_gain); break; case BBR_DRAIN: - bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */ - bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */ + bbr->pacing_gain = bbr_param(sk, drain_gain); /* slow, to drain */ + bbr->cwnd_gain = bbr_param(sk, startup_cwnd_gain); /* keep cwnd */ break; case BBR_PROBE_BW: - bbr->pacing_gain = (bbr->lt_use_bw ? - BBR_UNIT : - bbr_pacing_gain[bbr->cycle_idx]); - bbr->cwnd_gain = bbr_cwnd_gain; + bbr->pacing_gain = bbr_pacing_gain[bbr->cycle_idx]; + bbr->cwnd_gain = bbr_param(sk, cwnd_gain); + if (bbr_param(sk, bw_probe_cwnd_gain) && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr->cwnd_gain += + BBR_UNIT * bbr_param(sk, bw_probe_cwnd_gain) / 4; break; case BBR_PROBE_RTT: bbr->pacing_gain = BBR_UNIT; @@ -1013,144 +972,1387 @@ static void bbr_update_gains(struct sock *sk) } } -static void bbr_update_model(struct sock *sk, const struct rate_sample *rs) +__bpf_kfunc static u32 bbr_sndbuf_expand(struct sock *sk) { - bbr_update_bw(sk, rs); - bbr_update_ack_aggregation(sk, rs); - bbr_update_cycle_phase(sk, rs); - bbr_check_full_bw_reached(sk, rs); - bbr_check_drain(sk, rs); - bbr_update_min_rtt(sk, rs); - bbr_update_gains(sk); + /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ + return 3; } -__bpf_kfunc static void bbr_main(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs) +/* Incorporate a new bw sample into the current window of our max filter. */ +static void bbr_take_max_bw_sample(struct sock *sk, u32 bw) { struct bbr *bbr = inet_csk_ca(sk); - u32 bw; - - bbr_update_model(sk, rs); - bw = bbr_bw(sk); - bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); - bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain); + bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]); } -__bpf_kfunc static void bbr_init(struct sock *sk) +/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */ +static void bbr_advance_max_bw_filter(struct sock *sk) { - struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - bbr->prior_cwnd = 0; - tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; - bbr->rtt_cnt = 0; - bbr->next_rtt_delivered = tp->delivered; - bbr->prev_ca_state = TCP_CA_Open; - bbr->packet_conservation = 0; - - bbr->probe_rtt_done_stamp = 0; - bbr->probe_rtt_round_done = 0; - bbr->min_rtt_us = tcp_min_rtt(tp); - bbr->min_rtt_stamp = tcp_jiffies32; - - minmax_reset(&bbr->bw, bbr->rtt_cnt, 0); /* init max bw to 0 */ + if (!bbr->bw_hi[1]) + return; /* no samples in this window; remember old window */ + bbr->bw_hi[0] = bbr->bw_hi[1]; + bbr->bw_hi[1] = 0; +} - bbr->has_seen_rtt = 0; - bbr_init_pacing_rate_from_rtt(sk); +/* Reset the estimator for reaching full bandwidth based on bw plateau. */ +static void bbr_reset_full_bw(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); - bbr->round_start = 0; - bbr->idle_restart = 0; - bbr->full_bw_reached = 0; bbr->full_bw = 0; bbr->full_bw_cnt = 0; - bbr->cycle_mstamp = 0; - bbr->cycle_idx = 0; - bbr_reset_lt_bw_sampling(sk); - bbr_reset_startup_mode(sk); + bbr->full_bw_now = 0; +} - bbr->ack_epoch_mstamp = tp->tcp_mstamp; - bbr->ack_epoch_acked = 0; - bbr->extra_acked_win_rtts = 0; - bbr->extra_acked_win_idx = 0; - bbr->extra_acked[0] = 0; - bbr->extra_acked[1] = 0; +/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */ +static u32 bbr_target_inflight(struct sock *sk) +{ + u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT); - cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + return min(bdp, tcp_sk(sk)->snd_cwnd); } -__bpf_kfunc static u32 bbr_sndbuf_expand(struct sock *sk) +static bool bbr_is_probing_bandwidth(struct sock *sk) { - /* Provision 3 * cwnd since BBR may slow-start even during recovery. */ - return 3; + struct bbr *bbr = inet_csk_ca(sk); + + return (bbr->mode == BBR_STARTUP) || + (bbr->mode == BBR_PROBE_BW && + (bbr->cycle_idx == BBR_BW_PROBE_REFILL || + bbr->cycle_idx == BBR_BW_PROBE_UP)); +} + +/* Has the given amount of time elapsed since we marked the phase start? */ +static bool bbr_has_elapsed_in_phase(const struct sock *sk, u32 interval_us) +{ + const struct tcp_sock *tp = tcp_sk(sk); + const struct bbr *bbr = inet_csk_ca(sk); + + return tcp_stamp_us_delta(tp->tcp_mstamp, + bbr->cycle_mstamp + interval_us) > 0; +} + +static void bbr_handle_queue_too_high_in_startup(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bdp; /* estimated BDP in packets, with quantization budget */ + + bbr->full_bw_reached = 1; + + bdp = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr->inflight_hi = max(bdp, bbr->inflight_latest); +} + +/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */ +static void bbr_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible || + !bbr_param(sk, full_ecn_cnt) || !bbr_param(sk, ecn_thresh)) + return; + + if (ce_ratio >= bbr_param(sk, ecn_thresh)) + bbr->startup_ecn_rounds++; + else + bbr->startup_ecn_rounds = 0; + + if (bbr->startup_ecn_rounds >= bbr_param(sk, full_ecn_cnt)) { + bbr_handle_queue_too_high_in_startup(sk); + return; + } +} + +/* Updates ecn_alpha and returns ce_ratio. -1 if not available. */ +static int bbr_update_ecn_alpha(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct net *net = sock_net(sk); + struct bbr *bbr = inet_csk_ca(sk); + s32 delivered, delivered_ce; + u64 alpha, ce_ratio; + u32 gain; + bool want_ecn_alpha; + + /* See if we should use ECN sender logic for this connection. */ + if (!bbr->ecn_eligible && bbr_can_use_ecn(sk) && + !!bbr_param(sk, ecn_factor) && + (bbr->min_rtt_us <= bbr_ecn_max_rtt_us || + !bbr_ecn_max_rtt_us)) + bbr->ecn_eligible = 1; + + /* Skip updating alpha only if not ECN-eligible and PLB is disabled. */ + want_ecn_alpha = (bbr->ecn_eligible || + (bbr_can_use_ecn(sk) && + READ_ONCE(net->ipv4.sysctl_tcp_plb_enabled))); + if (!want_ecn_alpha) + return -1; + + delivered = tp->delivered - bbr->alpha_last_delivered; + delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce; + + if (delivered == 0 || /* avoid divide by zero */ + WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */ + return -1; + + BUILD_BUG_ON(BBR_SCALE != TCP_PLB_SCALE); + ce_ratio = (u64)delivered_ce << BBR_SCALE; + do_div(ce_ratio, delivered); + + gain = bbr_param(sk, ecn_alpha_gain); + alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE; + alpha += (gain * ce_ratio) >> BBR_SCALE; + bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT); + + bbr->alpha_last_delivered = tp->delivered; + bbr->alpha_last_delivered_ce = tp->delivered_ce; + + bbr_check_ecn_too_high_in_startup(sk, ce_ratio); + return (int)ce_ratio; } -/* In theory BBR does not need to undo the cwnd since it does not - * always reduce cwnd on losses (see bbr_main()). Keep it for now. +/* Protective Load Balancing (PLB). PLB rehashes outgoing data (to a new IPv6 + * flow label) if it encounters sustained congestion in the form of ECN marks. */ -__bpf_kfunc static u32 bbr_undo_cwnd(struct sock *sk) +static void bbr_plb(struct sock *sk, const struct rate_sample *rs, int ce_ratio) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->round_start && ce_ratio >= 0) + tcp_plb_update_state(sk, &bbr->plb, ce_ratio); + + tcp_plb_check_rehash(sk, &bbr->plb); +} + +/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */ +static void bbr_raise_inflight_hi_slope(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 growth_this_round, cnt; + + /* Calculate "slope": packets S/Acked per inflight_hi increment. */ + growth_this_round = 1 << bbr->bw_probe_up_rounds; + bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30); + cnt = tcp_snd_cwnd(tp) / growth_this_round; + cnt = max(cnt, 1U); + bbr->bw_probe_up_cnt = cnt; +} + +/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */ +static void bbr_probe_inflight_hi_upward(struct sock *sk, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 delta; + + if (!tp->is_cwnd_limited || tcp_snd_cwnd(tp) < bbr->inflight_hi) + return; /* not fully using inflight_hi, so don't grow it */ + + /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */ + bbr->bw_probe_up_acks += rs->acked_sacked; + if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) { + delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt; + bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt; + bbr->inflight_hi += delta; + bbr->try_fast_path = 0; /* Need to update cwnd */ + } + + if (bbr->round_start) + bbr_raise_inflight_hi_slope(sk); +} + +/* Does loss/ECN rate for this sample say inflight is "too high"? + * This is used by both the bbr_check_loss_too_high_in_startup() function, + * and in PROBE_UP. + */ +static bool bbr_is_inflight_too_high(const struct sock *sk, + const struct rate_sample *rs) +{ + const struct bbr *bbr = inet_csk_ca(sk); + u32 loss_thresh, ecn_thresh; + + if (rs->lost > 0 && rs->tx_in_flight) { + loss_thresh = (u64)rs->tx_in_flight * bbr_param(sk, loss_thresh) >> + BBR_SCALE; + if (rs->lost > loss_thresh) { + return true; + } + } + + if (rs->delivered_ce > 0 && rs->delivered > 0 && + bbr->ecn_eligible && !!bbr_param(sk, ecn_thresh)) { + ecn_thresh = (u64)rs->delivered * bbr_param(sk, ecn_thresh) >> + BBR_SCALE; + if (rs->delivered_ce > ecn_thresh) { + return true; + } + } + + return false; +} + +/* Calculate the tx_in_flight level that corresponded to excessive loss. + * We find "lost_prefix" segs of the skb where loss rate went too high, + * by solving for "lost_prefix" in the following equation: + * lost / inflight >= loss_thresh + * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh + * Then we take that equation, convert it to fixed point, and + * round up to the nearest packet. + */ +static u32 bbr_inflight_hi_from_lost_skb(const struct sock *sk, + const struct rate_sample *rs, + const struct sk_buff *skb) +{ + const struct tcp_sock *tp = tcp_sk(sk); + u32 loss_thresh = bbr_param(sk, loss_thresh); + u32 pcount, divisor, inflight_hi; + s32 inflight_prev, lost_prev; + u64 loss_budget, lost_prefix; + + pcount = tcp_skb_pcount(skb); + + /* How much data was in flight before this skb? */ + inflight_prev = rs->tx_in_flight - pcount; + if (inflight_prev < 0) { + WARN_ONCE(tcp_skb_tx_in_flight_is_suspicious( + pcount, + TCP_SKB_CB(skb)->sacked, + rs->tx_in_flight), + "tx_in_flight: %u pcount: %u reneg: %u", + rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg); + return ~0U; + } + + /* How much inflight data was marked lost before this skb? */ + lost_prev = rs->lost - pcount; + if (WARN_ONCE(lost_prev < 0, + "cwnd: %u ca: %d out: %u lost: %u pif: %u " + "tx_in_flight: %u tx.lost: %u tp->lost: %u rs->lost: %d " + "lost_prev: %d pcount: %d seq: %u end_seq: %u reneg: %u", + tcp_snd_cwnd(tp), inet_csk(sk)->icsk_ca_state, + tp->packets_out, tp->lost_out, tcp_packets_in_flight(tp), + rs->tx_in_flight, TCP_SKB_CB(skb)->tx.lost, tp->lost, + rs->lost, lost_prev, pcount, + TCP_SKB_CB(skb)->seq, TCP_SKB_CB(skb)->end_seq, + tp->is_sack_reneg)) + return ~0U; + + /* At what prefix of this lost skb did losss rate exceed loss_thresh? */ + loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1; + loss_budget >>= BBR_SCALE; + if (lost_prev >= loss_budget) { + lost_prefix = 0; /* previous losses crossed loss_thresh */ + } else { + lost_prefix = loss_budget - lost_prev; + lost_prefix <<= BBR_SCALE; + divisor = BBR_UNIT - loss_thresh; + if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */ + return ~0U; + do_div(lost_prefix, divisor); + } + + inflight_hi = inflight_prev + lost_prefix; + return inflight_hi; +} + +/* If loss/ECN rates during probing indicated we may have overfilled a + * buffer, return an operating point that tries to leave unutilized headroom in + * the path for other flows, for fairness convergence and lower RTTs and loss. + */ +static u32 bbr_inflight_with_headroom(const struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 headroom, headroom_fraction; + + if (bbr->inflight_hi == ~0U) + return ~0U; + + headroom_fraction = bbr_param(sk, inflight_headroom); + headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE; + headroom = max(headroom, 1U); + return max_t(s32, bbr->inflight_hi - headroom, + bbr_param(sk, cwnd_min_target)); +} + +/* Bound cwnd to a sensible level, based on our current probing state + * machine phase and model of a good inflight level (inflight_lo, inflight_hi). + */ +static void bbr_bound_cwnd_for_inflight_model(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + u32 cap; + + /* tcp_rcv_synsent_state_process() currently calls tcp_ack() + * and thus cong_control() without first initializing us(!). + */ + if (!bbr->initialized) + return; + + cap = ~0U; + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx != BBR_BW_PROBE_CRUISE) { + /* Probe to see if more packets fit in the path. */ + cap = bbr->inflight_hi; + } else { + if (bbr->mode == BBR_PROBE_RTT || + (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_CRUISE)) + cap = bbr_inflight_with_headroom(sk); + } + /* Adapt to any loss/ECN since our last bw probe. */ + cap = min(cap, bbr->inflight_lo); + + cap = max_t(u32, cap, bbr_param(sk, cwnd_min_target)); + tcp_snd_cwnd_set(tp, min(cap, tcp_snd_cwnd(tp))); +} + +/* How should we multiplicatively cut bw or inflight limits based on ECN? */ +static u32 bbr_ecn_cut(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + return BBR_UNIT - + ((bbr->ecn_alpha * bbr_param(sk, ecn_factor)) >> BBR_SCALE); +} + +/* Init lower bounds if have not inited yet. */ +static void bbr_init_lower_bounds(struct sock *sk, bool init_bw) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + if (init_bw && bbr->bw_lo == ~0U) + bbr->bw_lo = bbr_max_bw(sk); + if (bbr->inflight_lo == ~0U) + bbr->inflight_lo = tcp_snd_cwnd(tp); +} + +/* Reduce bw and inflight to (1 - beta). */ +static void bbr_loss_lower_bounds(struct sock *sk, u32 *bw, u32 *inflight) +{ + struct bbr* bbr = inet_csk_ca(sk); + u32 loss_cut = BBR_UNIT - bbr_param(sk, beta); + + *bw = max_t(u32, bbr->bw_latest, + (u64)bbr->bw_lo * loss_cut >> BBR_SCALE); + *inflight = max_t(u32, bbr->inflight_latest, + (u64)bbr->inflight_lo * loss_cut >> BBR_SCALE); +} + +/* Reduce inflight to (1 - alpha*ecn_factor). */ +static void bbr_ecn_lower_bounds(struct sock *sk, u32 *inflight) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_cut = bbr_ecn_cut(sk); + + *inflight = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE; +} + +/* Estimate a short-term lower bound on the capacity available now, based + * on measurements of the current delivery process and recent history. When we + * are seeing loss/ECN at times when we are not probing bw, then conservatively + * move toward flow balance by multiplicatively cutting our short-term + * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a + * multiplicative decrease in order to converge to a lower capacity in time + * logarithmic in the magnitude of the decrease. + * + * However, we do not cut our short-term estimates lower than the current rate + * and volume of delivered data from this round trip, since from the current + * delivery process we can estimate the measured capacity available now. + * + * Anything faster than that approach would knowingly risk high loss, which can + * cause low bw for Reno/CUBIC and high loss recovery latency for + * request/response flows using any congestion control. + */ +static void bbr_adapt_lower_bounds(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 ecn_inflight_lo = ~0U; + + /* We only use lower-bound estimates when not probing bw. + * When probing we need to push inflight higher to probe bw. + */ + if (bbr_is_probing_bandwidth(sk)) + return; + + /* ECN response. */ + if (bbr->ecn_in_round && !!bbr_param(sk, ecn_factor)) { + bbr_init_lower_bounds(sk, false); + bbr_ecn_lower_bounds(sk, &ecn_inflight_lo); + } + + /* Loss response. */ + if (bbr->loss_in_round) { + bbr_init_lower_bounds(sk, true); + bbr_loss_lower_bounds(sk, &bbr->bw_lo, &bbr->inflight_lo); + } + + /* Adjust to the lower of the levels implied by loss/ECN. */ + bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo); + bbr->bw_lo = max(1U, bbr->bw_lo); +} + +/* Reset any short-term lower-bound adaptation to congestion, so that we can + * push our inflight up. + */ +static void bbr_reset_lower_bounds(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->bw_lo = ~0U; + bbr->inflight_lo = ~0U; +} + +/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state + * machine phase where we adapt our lower bound based on congestion signals. + */ +static void bbr_reset_congestion_signals(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; + bbr->loss_in_cycle = 0; + bbr->ecn_in_cycle = 0; + bbr->bw_latest = 0; + bbr->inflight_latest = 0; +} + +static void bbr_exit_loss_recovery(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd)); + bbr->try_fast_path = 0; /* bound cwnd using latest model */ +} + +/* Update rate and volume of delivered data from latest round trip. */ +static void bbr_update_latest_delivery_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->loss_round_start = 0; + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + + bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw); + bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered); + + if (!before(rs->prior_delivered, bbr->loss_round_delivered)) { + bbr->loss_round_delivered = tp->delivered; + bbr->loss_round_start = 1; /* mark start of new round trip */ + } +} + +/* Once per round, reset filter for latest rate and volume of delivered data. */ +static void bbr_advance_latest_delivery_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* If ACK matches a TLP retransmit, persist the filter. If we detect + * that a TLP retransmit plugged a tail loss, we'll want to remember + * how much data the path delivered before the tail loss. + */ + if (bbr->loss_round_start && !rs->is_acking_tlp_retrans_seq) { + bbr->bw_latest = ctx->sample_bw; + bbr->inflight_latest = rs->delivered; + } +} + +/* Update (most of) our congestion signals: track the recent rate and volume of + * delivered data, presence of loss, and EWMA degree of ECN marking. + */ +static void bbr_update_congestion_signals( + struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u64 bw; + + if (rs->interval_us <= 0 || !rs->acked_sacked) + return; /* Not a valid observation */ + bw = ctx->sample_bw; + + if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) + bbr_take_max_bw_sample(sk, bw); + + bbr->loss_in_round |= (rs->losses > 0); + + if (!bbr->loss_round_start) + return; /* skip the per-round-trip updates */ + /* Now do per-round-trip updates. */ + bbr_adapt_lower_bounds(sk, rs); + + bbr->loss_in_round = 0; + bbr->ecn_in_round = 0; +} + +/* Bandwidth probing can cause loss. To help coexistence with loss-based + * congestion control we spread out our probing in a Reno-conscious way. Due to + * the shape of the Reno sawtooth, the time required between loss epochs for an + * idealized Reno flow is a number of round trips that is the BDP of that + * flow. We count packet-timed round trips directly, since measured RTT can + * vary widely, and Reno is driven by packet-timed round trips. + */ +static bool bbr_is_reno_coexistence_probe_time(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 rounds; + + /* Random loss can shave some small percentage off of our inflight + * in each round. To survive this, flows need robust periodic probes. + */ + rounds = min_t(u32, bbr_param(sk, bw_probe_max_rounds), bbr_target_inflight(sk)); + return bbr->rounds_since_probe >= rounds; +} + +/* How long do we want to wait before probing for bandwidth (and risking + * loss)? We randomize the wait, for better mixing and fairness convergence. + * + * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips. + * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow, + * (eg 4K video to a broadband user): + * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets + * + * We bound the BBR-native inter-bw-probe wall clock time to be: + * (a) higher than 2 sec: to try to avoid causing loss for a long enough time + * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must + * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs + * (b) lower than 3 sec: to ensure flows can start probing in a reasonable + * amount of time to discover unutilized bw on human-scale interactive + * time-scales (e.g. perhaps traffic from a web page download that we + * were competing with is now complete). + */ +static void bbr_pick_probe_wait(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Decide the random round-trip bound for wait until probe: */ + bbr->rounds_since_probe = + get_random_u32_below(bbr_param(sk, bw_probe_rand_rounds)); + /* Decide the random wall clock bound for wait until probe: */ + bbr->probe_wait_us = bbr_param(sk, bw_probe_base_us) + + get_random_u32_below(bbr_param(sk, bw_probe_rand_us)); +} + +static void bbr_set_cycle_idx(struct sock *sk, int cycle_idx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr->cycle_idx = cycle_idx; + /* New phase, so need to update cwnd and pacing rate. */ + bbr->try_fast_path = 0; +} + +/* Send at estimated bw to fill the pipe, but not queue. We need this phase + * before PROBE_UP, because as soon as we send faster than the available bw + * we will start building a queue, and if the buffer is shallow we can cause + * loss. If we do not fill the pipe before we cause this loss, our bw_hi and + * inflight_hi estimates will underestimate. + */ +static void bbr_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_lower_bounds(sk); + bbr->bw_probe_up_rounds = bw_probe_up_rounds; + bbr->bw_probe_up_acks = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_REFILLING; + bbr->next_rtt_delivered = tp->delivered; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_REFILL); +} + +/* Now probe max deliverable data rate and volume. */ +static void bbr_start_bw_probe_up(struct sock *sk, struct bbr_context *ctx) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->ack_phase = BBR_ACKS_PROBE_STARTING; + bbr->next_rtt_delivered = tp->delivered; + bbr->cycle_mstamp = tp->tcp_mstamp; + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_UP); + bbr_raise_inflight_hi_slope(sk); +} + +/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall + * clock time at which to probe beyond an inflight that we think to be + * safe. This will knowingly risk packet loss, so we want to do this rarely, to + * keep packet loss rates low. Also start a round-trip counter, to probe faster + * if we estimate a Reno flow at our BDP would probe faster. + */ +static void bbr_start_bw_probe_down(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_congestion_signals(sk); + bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */ + bbr_pick_probe_wait(sk); + bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */ + bbr->ack_phase = BBR_ACKS_PROBE_STOPPING; + bbr->next_rtt_delivered = tp->delivered; + bbr_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); +} + +/* Cruise: maintain what we estimate to be a neutral, conservative + * operating point, without attempting to probe up for bandwidth or down for + * RTT, and only reducing inflight in response to loss/ECN signals. + */ +static void bbr_start_bw_probe_cruise(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->inflight_lo != ~0U) + bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi); + + bbr_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); +} + +/* Loss and/or ECN rate is too high while probing. + * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle. + */ +static void bbr_handle_inflight_too_high(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + const u32 beta = bbr_param(sk, beta); + + bbr->prev_probe_too_high = 1; + bbr->bw_probe_samples = 0; /* only react once per probe */ + /* If we are app-limited then we are not robustly + * probing the max volume of inflight data we think + * might be safe (analogous to how app-limited bw + * samples are not known to be robustly probing bw). + */ + if (!rs->is_app_limited) { + bbr->inflight_hi = max_t(u32, rs->tx_in_flight, + (u64)bbr_target_inflight(sk) * + (BBR_UNIT - beta) >> BBR_SCALE); + } + if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr_start_bw_probe_down(sk); +} + +/* If we're seeing bw and loss samples reflecting our bw probing, adapt + * using the signals we see. If loss or ECN mark rate gets too high, then adapt + * inflight_hi downward. If we're able to push inflight higher without such + * signals, push higher: adapt inflight_hi upward. + */ +static bool bbr_adapt_upper_bounds(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + /* Track when we'll see bw/loss samples resulting from our bw probes. */ + if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start) + bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK; + if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) { + /* End of samples from bw probing phase. */ + bbr->bw_probe_samples = 0; + bbr->ack_phase = BBR_ACKS_INIT; + /* At this point in the cycle, our current bw sample is also + * our best recent chance at finding the highest available bw + * for this flow. So now is the best time to forget the bw + * samples from the previous cycle, by advancing the window. + */ + if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited) + bbr_advance_max_bw_filter(sk); + /* If we had an inflight_hi, then probed and pushed inflight all + * the way up to hit that inflight_hi without seeing any + * high loss/ECN in all the resulting ACKs from that probing, + * then probe up again, this time letting inflight persist at + * inflight_hi for a round trip, then accelerating beyond. + */ + if (bbr->mode == BBR_PROBE_BW && + bbr->stopped_risky_probe && !bbr->prev_probe_too_high) { + bbr_start_bw_probe_refill(sk, 0); + return true; /* yes, decided state transition */ + } + } + if (bbr_is_inflight_too_high(sk, rs)) { + if (bbr->bw_probe_samples) /* sample is from bw probing? */ + bbr_handle_inflight_too_high(sk, rs); + } else { + /* Loss/ECN rate is declared safe. Adjust upper bound upward. */ + + if (bbr->inflight_hi == ~0U) + return false; /* no excess queue signals yet */ + + /* To be resilient to random loss, we must raise bw/inflight_hi + * if we observe in any phase that a higher level is safe. + */ + if (rs->tx_in_flight > bbr->inflight_hi) { + bbr->inflight_hi = rs->tx_in_flight; + } + + if (bbr->mode == BBR_PROBE_BW && + bbr->cycle_idx == BBR_BW_PROBE_UP) + bbr_probe_inflight_hi_upward(sk, rs); + } + + return false; +} + +/* Check if it's time to probe for bandwidth now, and if so, kick it off. */ +static bool bbr_check_time_to_probe_bw(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 n; + + /* If we seem to be at an operating point where we are not seeing loss + * but we are seeing ECN marks, then when the ECN marks cease we reprobe + * quickly (in case cross-traffic has ceased and freed up bw). + */ + if (bbr_param(sk, ecn_reprobe_gain) && bbr->ecn_eligible && + bbr->ecn_in_cycle && !bbr->loss_in_cycle && + inet_csk(sk)->icsk_ca_state == TCP_CA_Open) { + /* Calculate n so that when bbr_raise_inflight_hi_slope() + * computes growth_this_round as 2^n it will be roughly the + * desired volume of data (inflight_hi*ecn_reprobe_gain). + */ + n = ilog2((((u64)bbr->inflight_hi * + bbr_param(sk, ecn_reprobe_gain)) >> BBR_SCALE)); + bbr_start_bw_probe_refill(sk, n); + return true; + } + + if (bbr_has_elapsed_in_phase(sk, bbr->probe_wait_us) || + bbr_is_reno_coexistence_probe_time(sk)) { + bbr_start_bw_probe_refill(sk, 0); + return true; + } + return false; +} + +/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */ +static bool bbr_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw) +{ + /* Always need to pull inflight down to leave headroom in queue. */ + if (inflight > bbr_inflight_with_headroom(sk)) + return false; + + return inflight <= bbr_inflight(sk, bw, BBR_UNIT); +} + +/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */ +static void bbr_update_cycle_phase(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) { + struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); + bool is_bw_probe_done = false; + u32 inflight, bw; + + if (!bbr_full_bw_reached(sk)) + return; + + /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */ + if (bbr_adapt_upper_bounds(sk, rs, ctx)) + return; /* already decided state transition */ + + if (bbr->mode != BBR_PROBE_BW) + return; + + inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight); + bw = bbr_max_bw(sk); + + switch (bbr->cycle_idx) { + /* First we spend most of our time cruising with a pacing_gain of 1.0, + * which paces at the estimated bw, to try to fully use the pipe + * without building queue. If we encounter loss/ECN marks, we adapt + * by slowing down. + */ + case BBR_BW_PROBE_CRUISE: + if (bbr_check_time_to_probe_bw(sk, rs)) + return; /* already decided state transition */ + break; + + /* After cruising, when it's time to probe, we first "refill": we send + * at the estimated bw to fill the pipe, before probing higher and + * knowingly risking overflowing the bottleneck buffer (causing loss). + */ + case BBR_BW_PROBE_REFILL: + if (bbr->round_start) { + /* After one full round trip of sending in REFILL, we + * start to see bw samples reflecting our REFILL, which + * may be putting too much data in flight. + */ + bbr->bw_probe_samples = 1; + bbr_start_bw_probe_up(sk, ctx); + } + break; - bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ + /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to + * probe for bw. If we have not seen loss/ECN, we try to raise inflight + * to at least pacing_gain*BDP; note that this may take more than + * min_rtt if min_rtt is small (e.g. on a LAN). + * + * We terminate PROBE_UP bandwidth probing upon any of the following: + * + * (1) We've pushed inflight up to hit the inflight_hi target set in the + * most recent previous bw probe phase. Thus we want to start + * draining the queue immediately because it's very likely the most + * recently sent packets will fill the queue and cause drops. + * (2) If inflight_hi has not limited bandwidth growth recently, and + * yet delivered bandwidth has not increased much recently + * (bbr->full_bw_now). + * (3) Loss filter says loss rate is "too high". + * (4) ECN filter says ECN mark rate is "too high". + * + * (1) (2) checked here, (3) (4) checked in bbr_is_inflight_too_high() + */ + case BBR_BW_PROBE_UP: + if (bbr->prev_probe_too_high && + inflight >= bbr->inflight_hi) { + bbr->stopped_risky_probe = 1; + is_bw_probe_done = true; + } else { + if (tp->is_cwnd_limited && + tcp_snd_cwnd(tp) >= bbr->inflight_hi) { + /* inflight_hi is limiting bw growth */ + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + } else if (bbr->full_bw_now) { + /* Plateau in estimated bw. Pipe looks full. */ + is_bw_probe_done = true; + } + } + if (is_bw_probe_done) { + bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */ + bbr_start_bw_probe_down(sk); /* restart w/ down */ + } + break; + + /* After probing in PROBE_UP, we have usually accumulated some data in + * the bottleneck buffer (if bw probing didn't find more bw). We next + * enter PROBE_DOWN to try to drain any excess data from the queue. To + * do this, we use a pacing_gain < 1.0. We hold this pacing gain until + * our inflight is less then that target cruising point, which is the + * minimum of (a) the amount needed to leave headroom, and (b) the + * estimated BDP. Once inflight falls to match the target, we estimate + * the queue is drained; persisting would underutilize the pipe. + */ + case BBR_BW_PROBE_DOWN: + if (bbr_check_time_to_probe_bw(sk, rs)) + return; /* already decided state transition */ + if (bbr_check_time_to_cruise(sk, inflight, bw)) + bbr_start_bw_probe_cruise(sk); + break; + + default: + WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx); + } +} + +/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */ +static void bbr_exit_probe_rtt(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_lower_bounds(sk); + if (bbr_full_bw_reached(sk)) { + bbr->mode = BBR_PROBE_BW; + /* Raising inflight after PROBE_RTT may cause loss, so reset + * the PROBE_BW clock and schedule the next bandwidth probe for + * a friendly and randomized future point in time. + */ + bbr_start_bw_probe_down(sk); + /* Since we are exiting PROBE_RTT, we know inflight is + * below our estimated BDP, so it is reasonable to cruise. + */ + bbr_start_bw_probe_cruise(sk); + } else { + bbr->mode = BBR_STARTUP; + } +} + +/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until + * the end of the round in recovery to get a good estimate of how many packets + * have been lost, and how many we need to drain with a low pacing rate. + */ +static void bbr_check_loss_too_high_in_startup(struct sock *sk, + const struct rate_sample *rs) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr_full_bw_reached(sk)) + return; + + /* For STARTUP exit, check the loss rate at the end of each round trip + * of Recovery episodes in STARTUP. We check the loss rate at the end + * of the round trip to filter out noisy/low loss and have a better + * sense of inflight (extent of loss), so we can drain more accurately. + */ + if (rs->losses && bbr->loss_events_in_round < 0xf) + bbr->loss_events_in_round++; /* update saturating counter */ + if (bbr_param(sk, full_loss_cnt) && bbr->loss_round_start && + inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery && + bbr->loss_events_in_round >= bbr_param(sk, full_loss_cnt) && + bbr_is_inflight_too_high(sk, rs)) { + bbr_handle_queue_too_high_in_startup(sk); + return; + } + if (bbr->loss_round_start) + bbr->loss_events_in_round = 0; +} + +/* Estimate when the pipe is full, using the change in delivery rate: BBR + * estimates bw probing filled the pipe if the estimated bw hasn't changed by + * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited + * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the + * higher rwin, 3: we get higher delivery rate samples. Or transient + * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar + * design goal, but uses delay and inter-ACK spacing instead of bandwidth. + */ +static void bbr_check_full_bw_reached(struct sock *sk, + const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 bw_thresh, full_cnt, thresh; + + if (bbr->full_bw_now || rs->is_app_limited) + return; + + thresh = bbr_param(sk, full_bw_thresh); + full_cnt = bbr_param(sk, full_bw_cnt); + bw_thresh = (u64)bbr->full_bw * thresh >> BBR_SCALE; + if (ctx->sample_bw >= bw_thresh) { + bbr_reset_full_bw(sk); + bbr->full_bw = ctx->sample_bw; + return; + } + if (!bbr->round_start) + return; + ++bbr->full_bw_cnt; + bbr->full_bw_now = bbr->full_bw_cnt >= full_cnt; + bbr->full_bw_reached |= bbr->full_bw_now; +} + +/* If pipe is probably full, drain the queue and then enter steady-state. */ +static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + /* Set ssthresh to export purely for monitoring, to signal + * completion of initial STARTUP by setting to a non- + * TCP_INFINITE_SSTHRESH value (ssthresh is not used by BBR). + */ + tcp_sk(sk)->snd_ssthresh = + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + bbr_reset_congestion_signals(sk); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) { + bbr->mode = BBR_PROBE_BW; + bbr_start_bw_probe_down(sk); + } +} + +static void bbr_update_model(struct sock *sk, const struct rate_sample *rs, + struct bbr_context *ctx) +{ + bbr_update_congestion_signals(sk, rs, ctx); + bbr_update_ack_aggregation(sk, rs); + bbr_check_loss_too_high_in_startup(sk, rs); + bbr_check_full_bw_reached(sk, rs, ctx); + bbr_check_drain(sk, rs, ctx); + bbr_update_cycle_phase(sk, rs, ctx); + bbr_update_min_rtt(sk, rs); +} + +/* Fast path for app-limited case. + * + * On each ack, we execute bbr state machine, which primarily consists of: + * 1) update model based on new rate sample, and + * 2) update control based on updated model or state change. + * + * There are certain workload/scenarios, e.g. app-limited case, where + * either we can skip updating model or we can skip update of both model + * as well as control. This provides signifcant softirq cpu savings for + * processing incoming acks. + * + * In case of app-limited, if there is no congestion (loss/ecn) and + * if observed bw sample is less than current estimated bw, then we can + * skip some of the computation in bbr state processing: + * + * - if there is no rtt/mode/phase change: In this case, since all the + * parameters of the network model are constant, we can skip model + * as well control update. + * + * - else we can skip rest of the model update. But we still need to + * update the control to account for the new rtt/mode/phase. + * + * Returns whether we can take fast path or not. + */ +static bool bbr_run_fast_path(struct sock *sk, bool *update_model, + const struct rate_sample *rs, struct bbr_context *ctx) +{ + struct bbr *bbr = inet_csk_ca(sk); + u32 prev_min_rtt_us, prev_mode; + + if (bbr_param(sk, fast_path) && bbr->try_fast_path && + rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) && + !bbr->loss_in_round && !bbr->ecn_in_round ) { + prev_mode = bbr->mode; + prev_min_rtt_us = bbr->min_rtt_us; + bbr_check_drain(sk, rs, ctx); + bbr_update_cycle_phase(sk, rs, ctx); + bbr_update_min_rtt(sk, rs); + + if (bbr->mode == prev_mode && + bbr->min_rtt_us == prev_min_rtt_us && + bbr->try_fast_path) { + return true; + } + + /* Skip model update, but control still needs to be updated */ + *update_model = false; + } + return false; +} + +__bpf_kfunc static void bbr_main(struct sock *sk, u32 ack, int flag, + const struct rate_sample *rs) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct bbr_context ctx = { 0 }; + bool update_model = true; + u32 bw, round_delivered; + int ce_ratio = -1; + + round_delivered = bbr_update_round_start(sk, rs, &ctx); + if (bbr->round_start) { + bbr->rounds_since_probe = + min_t(s32, bbr->rounds_since_probe + 1, 0xFF); + ce_ratio = bbr_update_ecn_alpha(sk); + } + bbr_plb(sk, rs, ce_ratio); + + bbr->ecn_in_round |= (bbr->ecn_eligible && rs->is_ece); + bbr_calculate_bw_sample(sk, rs, &ctx); + bbr_update_latest_delivery_signals(sk, rs, &ctx); + + if (bbr_run_fast_path(sk, &update_model, rs, &ctx)) + goto out; + + if (update_model) + bbr_update_model(sk, rs, &ctx); + + bbr_update_gains(sk); + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain, + tcp_snd_cwnd(tp), &ctx); + bbr_bound_cwnd_for_inflight_model(sk); + +out: + bbr_advance_latest_delivery_signals(sk, rs, &ctx); + bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state; + bbr->loss_in_cycle |= rs->lost > 0; + bbr->ecn_in_cycle |= rs->delivered_ce > 0; +} + +__bpf_kfunc static void bbr_init(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + bbr->initialized = 1; + + bbr->init_cwnd = min(0x7FU, tcp_snd_cwnd(tp)); + bbr->prior_cwnd = tp->prior_cwnd; + tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + bbr->next_rtt_delivered = tp->delivered; + bbr->prev_ca_state = TCP_CA_Open; + + bbr->probe_rtt_done_stamp = 0; + bbr->probe_rtt_round_done = 0; + bbr->probe_rtt_min_us = tcp_min_rtt(tp); + bbr->probe_rtt_min_stamp = tcp_jiffies32; + bbr->min_rtt_us = tcp_min_rtt(tp); + bbr->min_rtt_stamp = tcp_jiffies32; + + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; + bbr->full_bw = 0; bbr->full_bw_cnt = 0; - bbr_reset_lt_bw_sampling(sk); - return tcp_snd_cwnd(tcp_sk(sk)); + bbr->cycle_mstamp = 0; + bbr->cycle_idx = 0; + + bbr_reset_startup_mode(sk); + + bbr->ack_epoch_mstamp = tp->tcp_mstamp; + bbr->ack_epoch_acked = 0; + bbr->extra_acked_win_rtts = 0; + bbr->extra_acked_win_idx = 0; + bbr->extra_acked[0] = 0; + bbr->extra_acked[1] = 0; + + bbr->ce_state = 0; + bbr->prior_rcv_nxt = tp->rcv_nxt; + bbr->try_fast_path = 0; + + cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + + /* Start sampling ECN mark rate after first full flight is ACKed: */ + bbr->loss_round_delivered = tp->delivered + 1; + bbr->loss_round_start = 0; + bbr->undo_bw_lo = 0; + bbr->undo_inflight_lo = 0; + bbr->undo_inflight_hi = 0; + bbr->loss_events_in_round = 0; + bbr->startup_ecn_rounds = 0; + bbr_reset_congestion_signals(sk); + bbr->bw_lo = ~0U; + bbr->bw_hi[0] = 0; + bbr->bw_hi[1] = 0; + bbr->inflight_lo = ~0U; + bbr->inflight_hi = ~0U; + bbr_reset_full_bw(sk); + bbr->bw_probe_up_cnt = ~0U; + bbr->bw_probe_up_acks = 0; + bbr->bw_probe_up_rounds = 0; + bbr->probe_wait_us = 0; + bbr->stopped_risky_probe = 0; + bbr->ack_phase = BBR_ACKS_INIT; + bbr->rounds_since_probe = 0; + bbr->bw_probe_samples = 0; + bbr->prev_probe_too_high = 0; + bbr->ecn_eligible = 0; + bbr->ecn_alpha = bbr_param(sk, ecn_alpha_init); + bbr->alpha_last_delivered = 0; + bbr->alpha_last_delivered_ce = 0; + bbr->plb.pause_until = 0; + + tp->fast_ack_mode = bbr_fast_ack_mode ? 1 : 0; + + if (bbr_can_use_ecn(sk)) + tp->ecn_flags |= TCP_ECN_ECT_PERMANENT; +} + +/* BBR marks the current round trip as a loss round. */ +static void bbr_note_loss(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + + /* Capture "current" data over the full round trip of loss, to + * have a better chance of observing the full capacity of the path. + */ + if (!bbr->loss_in_round) /* first loss in this round trip? */ + bbr->loss_round_delivered = tp->delivered; /* set round trip */ + bbr->loss_in_round = 1; + bbr->loss_in_cycle = 1; } -/* Entering loss recovery, so save cwnd for when we exit or undo recovery. */ +/* Core TCP stack informs us that the given skb was just marked lost. */ +__bpf_kfunc static void bbr_skb_marked_lost(struct sock *sk, + const struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct tcp_skb_cb *scb = TCP_SKB_CB(skb); + struct rate_sample rs = {}; + + bbr_note_loss(sk); + + if (!bbr->bw_probe_samples) + return; /* not an skb sent while probing for bandwidth */ + if (unlikely(!scb->tx.delivered_mstamp)) + return; /* skb was SACKed, reneged, marked lost; ignore it */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this lost skb, + * then see if the loss rate went too high, and if so at which packet. + */ + rs.tx_in_flight = scb->tx.in_flight; + rs.lost = tp->lost - scb->tx.lost; + rs.is_app_limited = scb->tx.is_app_limited; + if (bbr_is_inflight_too_high(sk, &rs)) { + rs.tx_in_flight = bbr_inflight_hi_from_lost_skb(sk, &rs, skb); + bbr_handle_inflight_too_high(sk, &rs); + } +} + +static void bbr_run_loss_probe_recovery(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); + struct rate_sample rs = {0}; + + bbr_note_loss(sk); + + if (!bbr->bw_probe_samples) + return; /* not sent while probing for bandwidth */ + /* We are probing for bandwidth. Construct a rate sample that + * estimates what happened in the flight leading up to this + * loss, then see if the loss rate went too high. + */ + rs.lost = 1; /* TLP probe repaired loss of a single segment */ + rs.tx_in_flight = bbr->inflight_latest + rs.lost; + rs.is_app_limited = tp->tlp_orig_data_app_limited; + if (bbr_is_inflight_too_high(sk, &rs)) + bbr_handle_inflight_too_high(sk, &rs); +} + +/* Revert short-term model if current loss recovery event was spurious. */ +__bpf_kfunc static u32 bbr_undo_cwnd(struct sock *sk) +{ + struct bbr *bbr = inet_csk_ca(sk); + + bbr_reset_full_bw(sk); /* spurious slow-down; reset full bw detector */ + bbr->loss_in_round = 0; + + /* Revert to cwnd and other state saved before loss episode. */ + bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo); + bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo); + bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi); + bbr->try_fast_path = 0; /* take slow path to set proper cwnd, pacing */ + return bbr->prior_cwnd; +} + +/* Entering loss recovery, so save state for when we undo recovery. */ __bpf_kfunc static u32 bbr_ssthresh(struct sock *sk) { + struct bbr *bbr = inet_csk_ca(sk); + bbr_save_cwnd(sk); + /* For undo, save state that adapts based on loss signal. */ + bbr->undo_bw_lo = bbr->bw_lo; + bbr->undo_inflight_lo = bbr->inflight_lo; + bbr->undo_inflight_hi = bbr->inflight_hi; return tcp_sk(sk)->snd_ssthresh; } +static enum tcp_bbr_phase bbr_get_phase(struct bbr *bbr) +{ + switch (bbr->mode) { + case BBR_STARTUP: + return BBR_PHASE_STARTUP; + case BBR_DRAIN: + return BBR_PHASE_DRAIN; + case BBR_PROBE_BW: + break; + case BBR_PROBE_RTT: + return BBR_PHASE_PROBE_RTT; + default: + return BBR_PHASE_INVALID; + } + switch (bbr->cycle_idx) { + case BBR_BW_PROBE_UP: + return BBR_PHASE_PROBE_BW_UP; + case BBR_BW_PROBE_DOWN: + return BBR_PHASE_PROBE_BW_DOWN; + case BBR_BW_PROBE_CRUISE: + return BBR_PHASE_PROBE_BW_CRUISE; + case BBR_BW_PROBE_REFILL: + return BBR_PHASE_PROBE_BW_REFILL; + default: + return BBR_PHASE_INVALID; + } +} + static size_t bbr_get_info(struct sock *sk, u32 ext, int *attr, - union tcp_cc_info *info) + union tcp_cc_info *info) { if (ext & (1 << (INET_DIAG_BBRINFO - 1)) || ext & (1 << (INET_DIAG_VEGASINFO - 1))) { - struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); - u64 bw = bbr_bw(sk); - - bw = bw * tp->mss_cache * USEC_PER_SEC >> BW_SCALE; - memset(&info->bbr, 0, sizeof(info->bbr)); - info->bbr.bbr_bw_lo = (u32)bw; - info->bbr.bbr_bw_hi = (u32)(bw >> 32); - info->bbr.bbr_min_rtt = bbr->min_rtt_us; - info->bbr.bbr_pacing_gain = bbr->pacing_gain; - info->bbr.bbr_cwnd_gain = bbr->cwnd_gain; + u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk)); + u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk)); + u64 bw_lo = bbr->bw_lo == ~0U ? + ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo); + struct tcp_bbr_info *bbr_info = &info->bbr; + + memset(bbr_info, 0, sizeof(*bbr_info)); + bbr_info->bbr_bw_lo = (u32)bw; + bbr_info->bbr_bw_hi = (u32)(bw >> 32); + bbr_info->bbr_min_rtt = bbr->min_rtt_us; + bbr_info->bbr_pacing_gain = bbr->pacing_gain; + bbr_info->bbr_cwnd_gain = bbr->cwnd_gain; + bbr_info->bbr_bw_hi_lsb = (u32)bw_hi; + bbr_info->bbr_bw_hi_msb = (u32)(bw_hi >> 32); + bbr_info->bbr_bw_lo_lsb = (u32)bw_lo; + bbr_info->bbr_bw_lo_msb = (u32)(bw_lo >> 32); + bbr_info->bbr_mode = bbr->mode; + bbr_info->bbr_phase = (__u8)bbr_get_phase(bbr); + bbr_info->bbr_version = (__u8)BBR_VERSION; + bbr_info->bbr_inflight_lo = bbr->inflight_lo; + bbr_info->bbr_inflight_hi = bbr->inflight_hi; + bbr_info->bbr_extra_acked = bbr_extra_acked(sk); *attr = INET_DIAG_BBRINFO; - return sizeof(info->bbr); + return sizeof(*bbr_info); } return 0; } __bpf_kfunc static void bbr_set_state(struct sock *sk, u8 new_state) { + struct tcp_sock *tp = tcp_sk(sk); struct bbr *bbr = inet_csk_ca(sk); if (new_state == TCP_CA_Loss) { - struct rate_sample rs = { .losses = 1 }; bbr->prev_ca_state = TCP_CA_Loss; - bbr->full_bw = 0; - bbr->round_start = 1; /* treat RTO like end of a round */ - bbr_lt_bw_sampling(sk, &rs); + tcp_plb_update_state_upon_rto(sk, &bbr->plb); + /* The tcp_write_timeout() call to sk_rethink_txhash() likely + * repathed this flow, so re-learn the min network RTT on the + * new path: + */ + bbr_reset_full_bw(sk); + if (!bbr_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) { + /* bbr_adapt_lower_bounds() needs cwnd before + * we suffered an RTO, to update inflight_lo: + */ + bbr->inflight_lo = + max(tcp_snd_cwnd(tp), bbr->prior_cwnd); + } + } else if (bbr->prev_ca_state == TCP_CA_Loss && + new_state != TCP_CA_Loss) { + bbr_exit_loss_recovery(sk); } } + static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = { - .flags = TCP_CONG_NON_RESTRICTED, + .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS, .name = "bbr", .owner = THIS_MODULE, .init = bbr_init, .cong_control = bbr_main, .sndbuf_expand = bbr_sndbuf_expand, + .skb_marked_lost = bbr_skb_marked_lost, .undo_cwnd = bbr_undo_cwnd, .cwnd_event = bbr_cwnd_event, .ssthresh = bbr_ssthresh, - .min_tso_segs = bbr_min_tso_segs, + .tso_segs = bbr_tso_segs, .get_info = bbr_get_info, .set_state = bbr_set_state, }; @@ -1159,10 +2361,11 @@ BTF_KFUNCS_START(tcp_bbr_check_kfunc_ids) BTF_ID_FLAGS(func, bbr_init) BTF_ID_FLAGS(func, bbr_main) BTF_ID_FLAGS(func, bbr_sndbuf_expand) +BTF_ID_FLAGS(func, bbr_skb_marked_lost) BTF_ID_FLAGS(func, bbr_undo_cwnd) BTF_ID_FLAGS(func, bbr_cwnd_event) BTF_ID_FLAGS(func, bbr_ssthresh) -BTF_ID_FLAGS(func, bbr_min_tso_segs) +BTF_ID_FLAGS(func, bbr_tso_segs) BTF_ID_FLAGS(func, bbr_set_state) BTF_KFUNCS_END(tcp_bbr_check_kfunc_ids) @@ -1195,5 +2398,12 @@ MODULE_AUTHOR("Van Jacobson "); MODULE_AUTHOR("Neal Cardwell "); MODULE_AUTHOR("Yuchung Cheng "); MODULE_AUTHOR("Soheil Hassas Yeganeh "); +MODULE_AUTHOR("Priyaranjan Jha "); +MODULE_AUTHOR("Yousuk Seung "); +MODULE_AUTHOR("Kevin Yang "); +MODULE_AUTHOR("Arjun Roy "); +MODULE_AUTHOR("David Morley "); + MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)"); +MODULE_VERSION(__stringify(BBR_VERSION)); diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c index df758adbb445..e98e5dbc050e 100644 --- a/net/ipv4/tcp_cong.c +++ b/net/ipv4/tcp_cong.c @@ -237,6 +237,7 @@ void tcp_init_congestion_control(struct sock *sk) struct inet_connection_sock *icsk = inet_csk(sk); tcp_sk(sk)->prior_ssthresh = 0; + tcp_sk(sk)->fast_ack_mode = 0; if (icsk->icsk_ca_ops->init) icsk->icsk_ca_ops->init(sk); if (tcp_ca_needs_ecn(sk)) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index a35018e2d0ba..b849d76b24da 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -381,7 +381,7 @@ static void tcp_data_ecn_check(struct sock *sk, const struct sk_buff *skb) tcp_enter_quickack_mode(sk, 2); break; case INET_ECN_CE: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_IS_CE); if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) { @@ -392,7 +392,7 @@ static void tcp_data_ecn_check(struct sock *sk, const struct sk_buff *skb) tp->ecn_flags |= TCP_ECN_SEEN; break; default: - if (tcp_ca_needs_ecn(sk)) + if (tcp_ca_wants_ce_events(sk)) tcp_ca_event(sk, CA_EVENT_ECN_NO_CE); tp->ecn_flags |= TCP_ECN_SEEN; break; @@ -1139,7 +1139,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb) */ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb) { + struct sock *sk = (struct sock *)tp; + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; + tp->lost += tcp_skb_pcount(skb); + if (ca_ops->skb_marked_lost) + ca_ops->skb_marked_lost(sk, skb); } void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb) @@ -1511,6 +1516,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev, WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount); tcp_skb_pcount_add(skb, -pcount); + /* Adjust tx.in_flight as pcount is shifted from skb to prev. */ + if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount, + "prev in_flight: %u skb in_flight: %u pcount: %u", + TCP_SKB_CB(prev)->tx.in_flight, + TCP_SKB_CB(skb)->tx.in_flight, + pcount)) + TCP_SKB_CB(skb)->tx.in_flight = 0; + else + TCP_SKB_CB(skb)->tx.in_flight -= pcount; + TCP_SKB_CB(prev)->tx.in_flight += pcount; + /* When we're adding to gso_segs == 1, gso_size will be zero, * in theory this shouldn't be necessary but as long as DSACK * code can come after this skb later on it's better to keep @@ -3848,7 +3864,8 @@ static int tcp_replace_ts_recent(struct tcp_sock *tp, u32 seq) /* This routine deals with acks during a TLP episode and ends an episode by * resetting tlp_high_seq. Ref: TLP algorithm in draft-ietf-tcpm-rack */ -static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) +static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag, + struct rate_sample *rs) { struct tcp_sock *tp = tcp_sk(sk); @@ -3865,6 +3882,7 @@ static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) /* ACK advances: there was a loss, so reduce cwnd. Reset * tlp_high_seq in tcp_init_cwnd_reduction() */ + tcp_ca_event(sk, CA_EVENT_TLP_RECOVERY); tcp_init_cwnd_reduction(sk); tcp_set_ca_state(sk, TCP_CA_CWR); tcp_end_cwnd_reduction(sk); @@ -3875,6 +3893,11 @@ static void tcp_process_tlp_ack(struct sock *sk, u32 ack, int flag) FLAG_NOT_DUP | FLAG_DATA_SACKED))) { /* Pure dupack: original and TLP probe arrived; no loss */ tp->tlp_high_seq = 0; + } else { + /* This ACK matches a TLP retransmit. We cannot yet tell if + * this ACK is for the original or the TLP retransmit. + */ + rs->is_acking_tlp_retrans_seq = 1; } } @@ -3994,6 +4017,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una; rs.prior_in_flight = tcp_packets_in_flight(tp); + tcp_rate_check_app_limited(sk); /* ts_recent update must be made after we are sure that the packet * is in window. @@ -4059,7 +4083,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) tcp_in_ack_event(sk, flag); if (tp->tlp_high_seq) - tcp_process_tlp_ack(sk, ack, flag); + tcp_process_tlp_ack(sk, ack, flag, &rs); if (tcp_ack_is_dubious(sk, flag)) { if (!(flag & (FLAG_SND_UNA_ADVANCED | @@ -4083,6 +4107,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) delivered = tcp_newly_delivered(sk, delivered, flag); lost = tp->lost - lost; /* freshly marked lost */ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + rs.is_ece = !!(flag & FLAG_ECE); tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate); tcp_cong_control(sk, ack, delivered, flag, sack_state.rate); tcp_xmit_recovery(sk, rexmit); @@ -4103,7 +4128,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) tcp_ack_probe(sk); if (tp->tlp_high_seq) - tcp_process_tlp_ack(sk, ack, flag); + tcp_process_tlp_ack(sk, ack, flag, &rs); return 1; old_ack: @@ -5782,13 +5807,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible) /* More than one full frame received... */ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss && + (tp->fast_ack_mode == 1 || /* ... and right edge of window advances far enough. * (tcp_recvmsg() will send ACK otherwise). * If application uses SO_RCVLOWAT, we want send ack now if * we have not received enough bytes to satisfy the condition. */ - (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || - __tcp_select_window(sk) >= tp->rcv_wnd)) || + (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat || + __tcp_select_window(sk) >= tp->rcv_wnd))) || /* We ACK each frame or... */ tcp_in_quickack_mode(sk) || /* Protocol state mandates a one-time immediate ACK */ diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index fb9349be36b8..3c53e39f8201 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -472,6 +472,8 @@ void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst) u32 ca_key = dst_metric(dst, RTAX_CC_ALGO); bool ca_got_dst = false; + tcp_set_ecn_low_from_dst(sk, dst); + if (ca_key != TCP_CA_UNSPEC) { const struct tcp_congestion_ops *ca; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 13295a59d22e..3effb6e51e96 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -339,10 +339,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk); bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 || tcp_ca_needs_ecn(sk) || bpf_needs_ecn; + const struct dst_entry *dst = __sk_dst_get(sk); if (!use_ecn) { - const struct dst_entry *dst = __sk_dst_get(sk); - if (dst && dst_feature(dst, RTAX_FEATURE_ECN)) use_ecn = true; } @@ -354,6 +353,9 @@ static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) INET_ECN_xmit(sk); + + if (dst) + tcp_set_ecn_low_from_dst(sk, dst); } } @@ -391,7 +393,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb, th->cwr = 1; skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; } - } else if (!tcp_ca_needs_ecn(sk)) { + } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) && + !tcp_ca_needs_ecn(sk)) { /* ACK or retransmitted segment: clear ECT|CE */ INET_ECN_dontxmit(sk); } @@ -1614,7 +1617,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, { struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *buff; - int old_factor; + int old_factor, inflight_prev; long limit; u16 flags; int nlen; @@ -1689,6 +1692,30 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, if (diff) tcp_adjust_pcount(sk, skb, diff); + + inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor; + if (inflight_prev < 0) { + WARN_ONCE(tcp_skb_tx_in_flight_is_suspicious( + old_factor, + TCP_SKB_CB(skb)->sacked, + TCP_SKB_CB(skb)->tx.in_flight), + "inconsistent: tx.in_flight: %u " + "old_factor: %d mss: %u sacked: %u " + "1st pcount: %d 2nd pcount: %d " + "1st len: %u 2nd len: %u ", + TCP_SKB_CB(skb)->tx.in_flight, old_factor, + mss_now, TCP_SKB_CB(skb)->sacked, + tcp_skb_pcount(skb), tcp_skb_pcount(buff), + skb->len, buff->len); + inflight_prev = 0; + } + /* Set 1st tx.in_flight as if 1st were sent by itself: */ + TCP_SKB_CB(skb)->tx.in_flight = inflight_prev + + tcp_skb_pcount(skb); + /* Set 2nd tx.in_flight with new 1st and 2nd pcounts: */ + TCP_SKB_CB(buff)->tx.in_flight = inflight_prev + + tcp_skb_pcount(skb) + + tcp_skb_pcount(buff); } /* Link BUFF into the send queue. */ @@ -2045,13 +2072,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) { const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; - u32 min_tso, tso_segs; - - min_tso = ca_ops->min_tso_segs ? - ca_ops->min_tso_segs(sk) : - READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); + u32 tso_segs; - tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + tso_segs = ca_ops->tso_segs ? + ca_ops->tso_segs(sk, mss_now) : + tcp_tso_autosize(sk, mss_now, + sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs); return min_t(u32, tso_segs, sk->sk_gso_max_segs); } @@ -2777,6 +2803,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, skb_set_delivery_time(skb, tp->tcp_wstamp_ns, SKB_CLOCK_MONOTONIC); list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue); tcp_init_tso_segs(skb, mss_now); + tcp_set_tx_in_flight(sk, skb); goto repair; /* Skip network transmission */ } @@ -2989,6 +3016,7 @@ void tcp_send_loss_probe(struct sock *sk) if (WARN_ON(!skb || !tcp_skb_pcount(skb))) goto rearm_timer; + tp->tlp_orig_data_app_limited = TCP_SKB_CB(skb)->tx.is_app_limited; if (__tcp_retransmit_skb(sk, skb, 1)) goto rearm_timer; diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c index a8f6d9d06f2e..8737f2134648 100644 --- a/net/ipv4/tcp_rate.c +++ b/net/ipv4/tcp_rate.c @@ -34,6 +34,24 @@ * ready to send in the write queue. */ +void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb) +{ + struct tcp_sock *tp = tcp_sk(sk); + u32 in_flight; + + /* Check, sanitize, and record packets in flight after skb was sent. */ + in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb); + if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX, + "insane in_flight %u cc %s mss %u " + "cwnd %u pif %u %u %u %u\n", + in_flight, inet_csk(sk)->icsk_ca_ops->name, + tp->mss_cache, tp->snd_cwnd, + tp->packets_out, tp->retrans_out, + tp->sacked_out, tp->lost_out)) + in_flight = TCPCB_IN_FLIGHT_MAX; + TCP_SKB_CB(skb)->tx.in_flight = in_flight; +} + /* Snapshot the current delivery information in the skb, to generate * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered(). */ @@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp; TCP_SKB_CB(skb)->tx.delivered = tp->delivered; TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce; + TCP_SKB_CB(skb)->tx.lost = tp->lost; TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0; + tcp_set_tx_in_flight(sk, skb); } /* When an skb is sacked or acked, we fill in the rate sample with the (prior) @@ -91,18 +111,21 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, if (!rs->prior_delivered || tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp, scb->end_seq, rs->last_end_seq)) { + rs->prior_lost = scb->tx.lost; rs->prior_delivered_ce = scb->tx.delivered_ce; rs->prior_delivered = scb->tx.delivered; rs->prior_mstamp = scb->tx.delivered_mstamp; rs->is_app_limited = scb->tx.is_app_limited; rs->is_retrans = scb->sacked & TCPCB_RETRANS; + rs->tx_in_flight = scb->tx.in_flight; rs->last_end_seq = scb->end_seq; /* Record send time of most recently ACKed packet: */ tp->first_tx_mstamp = tx_tstamp; /* Find the duration of the "send phase" of this window: */ - rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp, - scb->tx.first_tx_mstamp); + rs->interval_us = tcp_stamp32_us_delta( + tp->first_tx_mstamp, + scb->tx.first_tx_mstamp); } /* Mark off the skb delivered once it's sacked to avoid being @@ -144,6 +167,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, return; } rs->delivered = tp->delivered - rs->prior_delivered; + rs->lost = tp->lost - rs->prior_lost; rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce; /* delivered_ce occupies less than 32 bits in the skb control block */ @@ -155,7 +179,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, * longer phase. */ snd_us = rs->interval_us; /* send phase */ - ack_us = tcp_stamp_us_delta(tp->tcp_mstamp, + ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp, rs->prior_mstamp); /* ack phase */ rs->interval_us = max(snd_us, ack_us); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index e4c616bbd727..e4a7a25d667d 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -565,7 +565,7 @@ void tcp_retransmit_timer(struct sock *sk) struct inet_sock *inet = inet_sk(sk); u32 rtx_delta; - rtx_delta = tcp_time_stamp_ts(tp) - (tp->retrans_stamp ?: + rtx_delta = tcp_time_stamp_ts(tp) - (tp->retrans_stamp ?: tcp_skb_timestamp_ts(tp->tcp_usec_ts, skb)); if (tp->tcp_usec_ts) rtx_delta /= USEC_PER_MSEC; @@ -702,6 +702,8 @@ void tcp_write_timer_handler(struct sock *sk) icsk_timeout(icsk)); return; } + + tcp_rate_check_app_limited(sk); tcp_mstamp_refresh(tcp_sk(sk)); event = icsk->icsk_pending; -- 2.49.0.654.g845c48a16a From d4afb98e41312bc8ca20b1879e1751416af90b0f Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:55:31 +0200 Subject: [PATCH 5/8] block Signed-off-by: Peter Jung --- block/bfq-iosched.c | 52 +++++++++++++++++++++++++++++++++++++++------ block/bfq-iosched.h | 12 +++++++++-- block/mq-deadline.c | 48 +++++++++++++++++++++++++++++++++++------ 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index abd80dc13562..cd06c79c4e92 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -467,6 +467,21 @@ static struct bfq_io_cq *bfq_bic_lookup(struct request_queue *q) return icq; } +static struct bfq_io_cq *bfq_bic_try_lookup(struct request_queue *q) +{ + if (!current->io_context) + return NULL; + if (spin_trylock_irq(&q->queue_lock)) { + struct bfq_io_cq *icq; + + icq = icq_to_bic(ioc_lookup_icq(q)); + spin_unlock_irq(&q->queue_lock); + return icq; + } + + return NULL; +} + /* * Scheduler run of queue, if there are requests pending and no one in the * driver that will restart queueing. @@ -2465,10 +2480,21 @@ static bool bfq_bio_merge(struct request_queue *q, struct bio *bio, * returned by bfq_bic_lookup does not go away before * bfqd->lock is taken. */ - struct bfq_io_cq *bic = bfq_bic_lookup(q); + struct bfq_io_cq *bic = bfq_bic_try_lookup(q); bool ret; - spin_lock_irq(&bfqd->lock); + /* + * bio merging is called for every bio queued, and it's very easy + * to run into contention because of that. If we fail getting + * the dd lock, just skip this merge attempt. For related IO, the + * plug will be the successful merging point. If we get here, we + * already failed doing the obvious merge. Chances of actually + * getting a merge off this path is a lot slimmer, so skipping an + * occassional lookup that will most likely not succeed anyway should + * not be a problem. + */ + if (!spin_trylock_irq(&bfqd->lock)) + return false; if (bic) { /* @@ -5317,6 +5343,18 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) struct bfq_queue *in_serv_queue; bool waiting_rq, idle_timer_disabled = false; + /* + * If someone else is already dispatching, skip this one. This will + * defer the next dispatch event to when something completes, and could + * potentially lower the queue depth for contended cases. + * + * See the logic in blk_mq_do_dispatch_sched(), which loops and + * retries if nothing is dispatched. + */ + if (test_bit(BFQ_DISPATCHING, &bfqd->run_state) || + test_and_set_bit_lock(BFQ_DISPATCHING, &bfqd->run_state)) + return NULL; + spin_lock_irq(&bfqd->lock); in_serv_queue = bfqd->in_service_queue; @@ -5328,6 +5366,7 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) waiting_rq && !bfq_bfqq_wait_request(in_serv_queue); } + clear_bit_unlock(BFQ_DISPATCHING, &bfqd->run_state); spin_unlock_irq(&bfqd->lock); bfq_update_dispatch_stats(hctx->queue, rq, idle_timer_disabled ? in_serv_queue : NULL, @@ -6250,10 +6289,9 @@ static inline void bfq_update_insert_stats(struct request_queue *q, static struct bfq_queue *bfq_init_rq(struct request *rq); -static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, +static void bfq_insert_request(struct request_queue *q, struct request *rq, blk_insert_t flags) { - struct request_queue *q = hctx->queue; struct bfq_data *bfqd = q->elevator->elevator_data; struct bfq_queue *bfqq; bool idle_timer_disabled = false; @@ -6315,7 +6353,7 @@ static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); - bfq_insert_request(hctx, rq, flags); + bfq_insert_request(hctx->queue, rq, flags); } } @@ -7254,6 +7292,8 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) q->elevator = eq; spin_unlock_irq(&q->queue_lock); + spin_lock_init(&bfqd->lock); + /* * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. * Grab a permanent reference to it, so that the normal code flow @@ -7371,8 +7411,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) /* see comments on the definition of next field inside bfq_data */ bfqd->actuator_load_threshold = 4; - spin_lock_init(&bfqd->lock); - /* * The invocation of the next bfq_create_group_hierarchy * function is the head of a chain of function calls diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index 687a3a7ba784..8589b58af79f 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -504,12 +504,22 @@ struct bfq_io_cq { unsigned int requests; /* Number of requests this process has in flight */ }; +enum { + BFQ_DISPATCHING = 0, +}; + /** * struct bfq_data - per-device data structure. * * All the fields are protected by @lock. */ struct bfq_data { + struct { + spinlock_t lock; + } ____cacheline_aligned_in_smp; + + unsigned long run_state; + /* device request queue */ struct request_queue *queue; /* dispatch queue */ @@ -795,8 +805,6 @@ struct bfq_data { /* fallback dummy bfqq for extreme OOM conditions */ struct bfq_queue oom_bfqq; - spinlock_t lock; - /* * bic associated with the task issuing current bio for * merging. This and the next field are used as a support to diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 754f6b7415cd..a5fa8f86178d 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -79,10 +79,20 @@ struct dd_per_prio { struct io_stats_per_prio stats; }; +enum { + DD_DISPATCHING = 0, +}; + struct deadline_data { /* * run time data */ + struct { + spinlock_t lock; + spinlock_t zone_lock; + } ____cacheline_aligned_in_smp; + + unsigned long run_state; struct dd_per_prio per_prio[DD_PRIO_COUNT]; @@ -100,8 +110,6 @@ struct deadline_data { int front_merges; u32 async_depth; int prio_aging_expire; - - spinlock_t lock; }; /* Maps an I/O priority class to a deadline scheduler priority. */ @@ -466,6 +474,18 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) struct request *rq; enum dd_prio prio; + /* + * If someone else is already dispatching, skip this one. This will + * defer the next dispatch event to when something completes, and could + * potentially lower the queue depth for contended cases. + * + * See the logic in blk_mq_do_dispatch_sched(), which loops and + * retries if nothing is dispatched. + */ + if (test_bit(DD_DISPATCHING, &dd->run_state) || + test_and_set_bit_lock(DD_DISPATCHING, &dd->run_state)) + return NULL; + spin_lock(&dd->lock); rq = dd_dispatch_prio_aged_requests(dd, now); if (rq) @@ -482,6 +502,7 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) } unlock: + clear_bit_unlock(DD_DISPATCHING, &dd->run_state); spin_unlock(&dd->lock); return rq; @@ -585,6 +606,9 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e) eq->elevator_data = dd; + spin_lock_init(&dd->lock); + spin_lock_init(&dd->zone_lock); + for (prio = 0; prio <= DD_PRIO_MAX; prio++) { struct dd_per_prio *per_prio = &dd->per_prio[prio]; @@ -601,7 +625,6 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e) dd->last_dir = DD_WRITE; dd->fifo_batch = fifo_batch; dd->prio_aging_expire = prio_aging_expire; - spin_lock_init(&dd->lock); /* We dispatch from request queue wide instead of hw queue */ blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); @@ -657,7 +680,19 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio, struct request *free = NULL; bool ret; - spin_lock(&dd->lock); + /* + * bio merging is called for every bio queued, and it's very easy + * to run into contention because of that. If we fail getting + * the dd lock, just skip this merge attempt. For related IO, the + * plug will be the successful merging point. If we get here, we + * already failed doing the obvious merge. Chances of actually + * getting a merge off this path is a lot slimmer, so skipping an + * occassional lookup that will most likely not succeed anyway should + * not be a problem. + */ + if (!spin_trylock(&dd->lock)) + return false; + ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); spin_unlock(&dd->lock); @@ -670,10 +705,9 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio, /* * add rq to rbtree and fifo */ -static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, +static void dd_insert_request(struct request_queue *q, struct request *rq, blk_insert_t flags, struct list_head *free) { - struct request_queue *q = hctx->queue; struct deadline_data *dd = q->elevator->elevator_data; const enum dd_data_dir data_dir = rq_data_dir(rq); u16 ioprio = req_get_ioprio(rq); @@ -731,7 +765,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); - dd_insert_request(hctx, rq, flags, &free); + dd_insert_request(q, rq, flags, &free); } spin_unlock(&dd->lock); -- 2.49.0.654.g845c48a16a From 3adf58d750bf5daa01d997ea56b715f1279a4b2b Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:55:42 +0200 Subject: [PATCH 6/8] cachy Signed-off-by: Peter Jung --- .gitignore | 3 + .../ABI/testing/sysfs-driver-intel-xe-hwmon | 24 + .../admin-guide/kernel-parameters.txt | 12 + Documentation/admin-guide/sysctl/vm.rst | 72 + MAINTAINERS | 5 + Makefile | 48 +- arch/Kconfig | 19 + arch/x86/Kconfig.cpu | 70 + arch/x86/Makefile | 17 + arch/x86/include/asm/pci.h | 6 + arch/x86/pci/common.c | 7 +- block/Kconfig.iosched | 14 + block/Makefile | 8 + block/adios.c | 1344 +++++++ block/elevator.c | 12 + drivers/Makefile | 13 +- drivers/ata/ahci.c | 23 +- drivers/cpufreq/Kconfig.x86 | 2 - drivers/cpufreq/intel_pstate.c | 2 + drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 10 + drivers/gpu/drm/amd/display/Kconfig | 6 + .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 2 +- .../amd/display/amdgpu_dm/amdgpu_dm_color.c | 2 +- .../amd/display/amdgpu_dm/amdgpu_dm_crtc.c | 6 +- .../amd/display/amdgpu_dm/amdgpu_dm_plane.c | 6 +- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 3 + drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 14 +- drivers/gpu/drm/xe/regs/xe_pcode_regs.h | 3 + drivers/gpu/drm/xe/xe_device_types.h | 2 + drivers/gpu/drm/xe/xe_hwmon.c | 125 +- drivers/gpu/drm/xe/xe_pci.c | 4 + drivers/gpu/drm/xe/xe_pcode_api.h | 3 + drivers/hid/amd-sfh-hid/amd_sfh_client.c | 2 + drivers/hid/amd-sfh-hid/amd_sfh_pcie.c | 4 + drivers/hid/amd-sfh-hid/amd_sfh_pcie.h | 1 + .../hid_descriptor/amd_sfh_hid_desc.c | 27 + .../hid_descriptor/amd_sfh_hid_desc.h | 8 + .../hid_descriptor/amd_sfh_hid_report_desc.h | 20 + drivers/input/evdev.c | 19 +- drivers/md/dm-crypt.c | 5 + drivers/media/v4l2-core/Kconfig | 5 + drivers/media/v4l2-core/Makefile | 2 + drivers/media/v4l2-core/v4l2loopback.c | 3292 +++++++++++++++++ drivers/media/v4l2-core/v4l2loopback.h | 98 + .../media/v4l2-core/v4l2loopback_formats.h | 445 +++ drivers/pci/controller/Makefile | 6 + drivers/pci/controller/intel-nvme-remap.c | 462 +++ drivers/pci/quirks.c | 101 + drivers/scsi/Kconfig | 2 + drivers/scsi/Makefile | 1 + drivers/scsi/vhba/Kconfig | 9 + drivers/scsi/vhba/Makefile | 4 + drivers/scsi/vhba/vhba.c | 1132 ++++++ fs/select.c | 2 +- include/linux/mm.h | 8 + include/linux/pagemap.h | 2 +- include/linux/user_namespace.h | 4 + include/linux/wait.h | 2 + init/Kconfig | 26 + kernel/Kconfig.hz | 24 + kernel/Kconfig.preempt | 2 +- kernel/fork.c | 14 + kernel/locking/rwsem.c | 4 +- kernel/sched/fair.c | 13 + kernel/sched/sched.h | 2 +- kernel/sched/wait.c | 24 + kernel/sysctl.c | 12 + kernel/user_namespace.c | 7 + mm/Kconfig | 65 +- mm/compaction.c | 4 + mm/huge_memory.c | 4 + mm/mm_init.c | 1 + mm/page-writeback.c | 8 + mm/page_alloc.c | 4 + mm/swap.c | 5 + mm/util.c | 34 + mm/vmpressure.c | 4 + mm/vmscan.c | 148 +- net/ipv4/inet_connection_sock.c | 2 +- scripts/Makefile.build | 52 +- scripts/Makefile.lib | 7 +- scripts/Makefile.vmlinux_o | 16 +- scripts/Makefile.vmlinux_thinlink | 53 + scripts/head-object-list.txt | 1 + 85 files changed, 8020 insertions(+), 67 deletions(-) create mode 100644 block/adios.c create mode 100644 drivers/media/v4l2-core/v4l2loopback.c create mode 100644 drivers/media/v4l2-core/v4l2loopback.h create mode 100644 drivers/media/v4l2-core/v4l2loopback_formats.h create mode 100644 drivers/pci/controller/intel-nvme-remap.c create mode 100644 drivers/scsi/vhba/Kconfig create mode 100644 drivers/scsi/vhba/Makefile create mode 100644 drivers/scsi/vhba/vhba.c create mode 100644 scripts/Makefile.vmlinux_thinlink diff --git a/.gitignore b/.gitignore index f2f63e47fb88..b83a68185ef4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ # .* *.a +*.a_thinlto_native *.asn1.[ch] *.bin *.bz2 @@ -39,6 +40,7 @@ *.mod.c *.o *.o.* +*.o_thinlto_native *.patch *.rmeta *.rpm @@ -64,6 +66,7 @@ modules.order /vmlinux /vmlinux.32 /vmlinux.map +/vmlinux.thinlink /vmlinux.symvers /vmlinux.unstripped /vmlinux-gdb.py diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon b/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon index cb207c79680d..6fbab98fb639 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon +++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon @@ -124,3 +124,27 @@ Contact: intel-xe@lists.freedesktop.org Description: RO. VRAM temperature in millidegree Celsius. Only supported for particular Intel Xe graphics platforms. + +What: /sys/bus/pci/drivers/xe/.../hwmon/hwmon/fan1_input +Date: March 2025 +KernelVersion: 6.14 +Contact: intel-xe@lists.freedesktop.org +Description: RO. Fan 1 speed in RPM. + + Only supported for particular Intel Xe graphics platforms. + +What: /sys/bus/pci/drivers/xe/.../hwmon/hwmon/fan2_input +Date: March 2025 +KernelVersion: 6.14 +Contact: intel-xe@lists.freedesktop.org +Description: RO. Fan 2 speed in RPM. + + Only supported for particular Intel Xe graphics platforms. + +What: /sys/bus/pci/drivers/xe/.../hwmon/hwmon/fan3_input +Date: March 2025 +KernelVersion: 6.14 +Contact: intel-xe@lists.freedesktop.org +Description: RO. Fan 3 speed in RPM. + + Only supported for particular Intel Xe graphics platforms. diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 8f75ec177399..c865280efa8e 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2318,6 +2318,9 @@ disable Do not enable intel_pstate as the default scaling driver for the supported processors + enable + Enable intel_pstate in-case "disable" was passed + previously in the kernel boot parameters active Use intel_pstate driver to bypass the scaling governors layer of cpufreq and provides it own @@ -4689,6 +4692,15 @@ nomsi [MSI] If the PCI_MSI kernel config parameter is enabled, this kernel boot option can be used to disable the use of MSI interrupts system-wide. + pcie_acs_override = + [PCIE] Override missing PCIe ACS support for: + downstream + All downstream ports - full ACS capabilities + multfunction + All multifunction devices - multifunction ACS subset + id:nnnn:nnnn + Specfic device - full ACS capabilities + Specified as vid:did (vendor/device ID) in hex noioapicquirk [APIC] Disable all boot interrupt quirks. Safety option to keep boot IRQs enabled. This should never be necessary. diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 8290177b4f75..f2e601171a3e 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -25,6 +25,9 @@ files can be found in mm/swap.c. Currently, these files are in /proc/sys/vm: - admin_reserve_kbytes +- anon_min_ratio +- clean_low_ratio +- clean_min_ratio - compact_memory - compaction_proactiveness - compact_unevictable_allowed @@ -109,6 +112,67 @@ On x86_64 this is about 128MB. Changing this takes effect whenever an application requests memory. +anon_min_ratio +============== + +This knob provides *hard* protection of anonymous pages. The anonymous pages +on the current node won't be reclaimed under any conditions when their amount +is below vm.anon_min_ratio. + +This knob may be used to prevent excessive swap thrashing when anonymous +memory is low (for example, when memory is going to be overfilled by +compressed data of zram module). + +Setting this value too high (close to 100) can result in inability to +swap and can lead to early OOM under memory pressure. + +The unit of measurement is the percentage of the total memory of the node. + +The default value is 15. + + +clean_low_ratio +================ + +This knob provides *best-effort* protection of clean file pages. The file pages +on the current node won't be reclaimed under memory pressure when the amount of +clean file pages is below vm.clean_low_ratio *unless* we threaten to OOM. + +Protection of clean file pages using this knob may be used when swapping is +still possible to + - prevent disk I/O thrashing under memory pressure; + - improve performance in disk cache-bound tasks under memory pressure. + +Setting it to a high value may result in a early eviction of anonymous pages +into the swap space by attempting to hold the protected amount of clean file +pages in memory. + +The unit of measurement is the percentage of the total memory of the node. + +The default value is 0. + + +clean_min_ratio +================ + +This knob provides *hard* protection of clean file pages. The file pages on the +current node won't be reclaimed under memory pressure when the amount of clean +file pages is below vm.clean_min_ratio. + +Hard protection of clean file pages using this knob may be used to + - prevent disk I/O thrashing under memory pressure even with no free swap space; + - improve performance in disk cache-bound tasks under memory pressure; + - avoid high latency and prevent livelock in near-OOM conditions. + +Setting it to a high value may result in a early out-of-memory condition due to +the inability to reclaim the protected amount of clean file pages when other +types of pages cannot be reclaimed. + +The unit of measurement is the percentage of the total memory of the node. + +The default value is 15. + + compact_memory ============== @@ -973,6 +1037,14 @@ be 133 (x + 2x = 200, 2x = 133.33). At 0, the kernel will not initiate swap until the amount of free and file-backed pages is less than the high watermark in a zone. +This knob has no effect if the amount of clean file pages on the current +node is below vm.clean_low_ratio or vm.clean_min_ratio. In this case, +only anonymous pages can be reclaimed. + +If the number of anonymous pages on the current node is below +vm.anon_min_ratio, then only file pages can be reclaimed with +any vm.swappiness value. + unprivileged_userfaultfd ======================== diff --git a/MAINTAINERS b/MAINTAINERS index dd844ac8d910..4feb3b9f8f73 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5790,6 +5790,11 @@ F: scripts/Makefile.clang F: scripts/clang-tools/ K: \b(?i:clang|llvm)\b +CLANG/LLVM THINLTO DISTRIBUTED BUILD +M: Rong Xu +S: Supported +F: scripts/Makefile.vmlinux_thinlink + CLK API M: Russell King L: linux-clk@vger.kernel.org diff --git a/Makefile b/Makefile index c1cd1b5fc269..5449040af65e 100644 --- a/Makefile +++ b/Makefile @@ -298,7 +298,8 @@ no-dot-config-targets := $(clean-targets) \ outputmakefile rustavailable rustfmt rustfmtcheck no-sync-config-targets := $(no-dot-config-targets) %install modules_sign kernelrelease \ image_name -single-targets := %.a %.i %.ko %.lds %.ll %.lst %.mod %.o %.rsi %.s %/ +single-targets := %.a %.i %.ko %.lds %.ll %.lst %.mod %.o %.rsi %.s %.o_thinlto_native \ + %.a_thinlto_native %.o.thinlto.bc %/ config-build := mixed-build := @@ -857,11 +858,19 @@ KBUILD_CFLAGS += -fno-delete-null-pointer-checks ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE KBUILD_CFLAGS += -O2 KBUILD_RUSTFLAGS += -Copt-level=2 +else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 +KBUILD_CFLAGS += -O3 +KBUILD_RUSTFLAGS += -Copt-level=3 else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE KBUILD_CFLAGS += -Os KBUILD_RUSTFLAGS += -Copt-level=s endif +# Perform swing modulo scheduling immediately before the first scheduling pass. +# This pass looks at innermost loops and reorders their instructions by +# overlapping different iterations. +KBUILD_CFLAGS += $(call cc-option,-fmodulo-sched -fmodulo-sched-allow-regmoves -fivopts -fmodulo-sched) + # Always set `debug-assertions` and `overflow-checks` because their default # depends on `opt-level` and `debug-assertions`, respectively. KBUILD_RUSTFLAGS += -Cdebug-assertions=$(if $(CONFIG_RUST_DEBUG_ASSERTIONS),y,n) @@ -991,10 +1000,10 @@ export CC_FLAGS_SCS endif ifdef CONFIG_LTO_CLANG -ifdef CONFIG_LTO_CLANG_THIN -CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit -else +ifdef CONFIG_LTO_CLANG_FULL CC_FLAGS_LTO := -flto +else # for CONFIG_LTO_CLANG_THIN or CONFIG_LTO_CLANG_THIN_DIST +CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit endif CC_FLAGS_LTO += -fvisibility=hidden @@ -1213,8 +1222,34 @@ vmlinux.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt FORCE $(call if_changed,ar_vmlinux.a) PHONY += vmlinux_o +ifdef CONFIG_LTO_CLANG_THIN_DIST +vmlinux.thinlink: vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_thinlink +targets += vmlinux.thinlink + +vmlinux.a_thinlto_native := $(patsubst %.a,%.a_thinlto_native,$(KBUILD_VMLINUX_OBJS)) +quiet_cmd_ar_vmlinux.a_thinlto_native = AR $@ + cmd_ar_vmlinux.a_thinlto_native = \ + rm -f $@; \ + $(AR) cDPrST $@ $(vmlinux.a_thinlto_native); \ + $(AR) mPiT $$($(AR) t $@ | sed -n 1p) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) + +define rule_gen_vmlinux.a_thinlto_native + +$(Q)$(MAKE) $(build)=. need-builtin=1 thinlto_final_pass=1 need-modorder=1 built-in.a_thinlto_native + $(call cmd_and_savecmd,ar_vmlinux.a_thinlto_native) +endef + +vmlinux.a_thinlto_native: vmlinux.thinlink scripts/head-object-list.txt FORCE + $(call if_changed_rule,gen_vmlinux.a_thinlto_native) + +targets += vmlinux.a_thinlto_native + +vmlinux_o: vmlinux.a_thinlto_native + $(Q)$(MAKE) thinlto_final_pass=1 -f $(srctree)/scripts/Makefile.vmlinux_o +else vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS) $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_o +endif vmlinux.o modules.builtin.modinfo modules.builtin: vmlinux_o @: @@ -1572,7 +1607,8 @@ CLEAN_FILES += vmlinux.symvers modules-only.symvers \ modules.builtin.ranges vmlinux.o.map vmlinux.unstripped \ compile_commands.json rust/test \ rust-project.json .vmlinux.objs .vmlinux.export.c \ - .builtin-dtbs-list .builtin-dtb.S + .builtin-dtbs-list .builtin-dtb.S \ + .vmlinux_thinlto_bc_files vmlinux.thinlink # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -2023,6 +2059,8 @@ clean: $(clean-dirs) -o -name '*.symtypes' -o -name 'modules.order' \ -o -name '*.c.[012]*.*' \ -o -name '*.ll' \ + -o -name '*.a_thinlto_native' -o -name '*.o_thinlto_native' \ + -o -name '*.o.thinlto.bc' \ -o -name '*.gcno' \ \) -type f -print \ -o -name '.tmp_*' -print \ diff --git a/arch/Kconfig b/arch/Kconfig index b0adb665041f..30dccda07c67 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -810,6 +810,25 @@ config LTO_CLANG_THIN https://clang.llvm.org/docs/ThinLTO.html If unsure, say Y. + +config LTO_CLANG_THIN_DIST + bool "Clang ThinLTO in distributed mode (EXPERIMENTAL)" + depends on HAS_LTO_CLANG && ARCH_SUPPORTS_LTO_CLANG_THIN + select LTO_CLANG + help + This option enables Clang's ThinLTO in distributed build mode. + In this mode, the linker performs the thin-link, generating + ThinLTO index files. Subsequently, the build system explicitly + invokes ThinLTO backend compilation using these index files + and pre-linked IR objects. The resulting native object files + are with the .o_thinlto_native suffix. + + This build mode offers improved visibility into the ThinLTO + process through explicit subcommand exposure. It also makes + final native object files directly available, benefiting + tools like objtool and kpatch. Additionally, it provides + crucial granular control over back-end options, enabling + module-specific compiler options, and simplifies debugging. endchoice config ARCH_SUPPORTS_AUTOFDO_CLANG diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu index 753b8763abae..d4ce964d9713 100644 --- a/arch/x86/Kconfig.cpu +++ b/arch/x86/Kconfig.cpu @@ -245,6 +245,76 @@ config MATOM endchoice +config CC_HAS_MARCH_NATIVE + # This flag might not be available in cross-compilers: + def_bool $(cc-option, -march=native) + # LLVM 18 has an easily triggered internal compiler error in core + # networking code with '-march=native' on certain systems: + # https://github.com/llvm/llvm-project/issues/72026 + # LLVM 19 introduces an optimization that resolves some high stack + # usage warnings that only appear wth '-march=native'. + depends on CC_IS_GCC || CLANG_VERSION >= 190100 + + +choice + prompt "x86_64 Compiler Build Optimization" + default GENERIC_CPU + +config X86_NATIVE_CPU + bool "Build and optimize for local/native CPU" + depends on X86_64 + depends on CC_HAS_MARCH_NATIVE + help + Optimize for the current CPU used to compile the kernel. + Use this option if you intend to build the kernel for your + local machine. + + Note that such a kernel might not work optimally on a + different x86 machine. + + If unsure, say N. + +config GENERIC_CPU + bool "Generic-x86-64" + depends on X86_64 + help + Generic x86-64 CPU. + Runs equally well on all x86-64 CPUs. + +config MZEN4 + bool "AMD Ryzen 4" + depends on (CC_IS_GCC && GCC_VERSION >= 130000) || (CC_IS_CLANG && CLANG_VERSION >= 160000) + help + Select this for AMD Family 19h Zen 4 processors. + + Enables -march=znver4 + +endchoice + +config X86_64_VERSION + int "x86-64 compiler ISA level" + range 1 4 + depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000) + depends on X86_64 && GENERIC_CPU + help + Specify a specific x86-64 compiler ISA level. + + There are three x86-64 ISA levels that work on top of + the x86-64 baseline, namely: x86-64-v2 and x86-64-v3. + + x86-64-v2 brings support for vector instructions up to Streaming SIMD + Extensions 4.2 (SSE4.2) and Supplemental Streaming SIMD Extensions 3 + (SSSE3), the POPCNT instruction, and CMPXCHG16B. + + x86-64-v3 adds vector instructions up to AVX2, MOVBE, and additional + bit-manipulation instructions. + + x86-64-v4 is not included since the kernel does not use AVX512 instructions + + You can find the best version for your CPU by running one of the following: + /lib/ld-linux-x86-64.so.2 --help | grep supported + /lib64/ld-linux-x86-64.so.2 --help | grep supported + config X86_GENERIC bool "Generic x86 support" depends on X86_32 diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 594723005d95..cbd234e9e743 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -173,8 +173,25 @@ else # Use -mskip-rax-setup if supported. KBUILD_CFLAGS += $(call cc-option,-mskip-rax-setup) +ifdef CONFIG_X86_NATIVE_CPU + KBUILD_CFLAGS += -march=native + KBUILD_RUSTFLAGS += -Ctarget-cpu=native +endif + +ifdef CONFIG_MZEN4 + KBUILD_CFLAGS += -march=znver4 + KBUILD_RUSTFLAGS += -Ctarget-cpu=znver4 +endif + +ifdef CONFIG_GENERIC_CPU +ifeq ($(CONFIG_X86_64_VERSION),1) KBUILD_CFLAGS += -march=x86-64 -mtune=generic KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64 -Ztune-cpu=generic +else + KBUILD_CFLAGS +=-march=x86-64-v$(CONFIG_X86_64_VERSION) + KBUILD_RUSTFLAGS += -Ctarget-cpu=x86-64-v$(CONFIG_X86_64_VERSION) +endif # CONFIG_X86_64_VERSION +endif # CONFIG_GENERIC_CPU KBUILD_CFLAGS += -mno-red-zone KBUILD_CFLAGS += -mcmodel=kernel diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h index b3ab80a03365..5e883b397ff3 100644 --- a/arch/x86/include/asm/pci.h +++ b/arch/x86/include/asm/pci.h @@ -26,6 +26,7 @@ struct pci_sysdata { #if IS_ENABLED(CONFIG_VMD) struct pci_dev *vmd_dev; /* VMD Device if in Intel VMD domain */ #endif + struct pci_dev *nvme_remap_dev; /* AHCI Device if NVME remapped bus */ }; extern int pci_routeirq; @@ -69,6 +70,11 @@ static inline bool is_vmd(struct pci_bus *bus) #define is_vmd(bus) false #endif /* CONFIG_VMD */ +static inline bool is_nvme_remap(struct pci_bus *bus) +{ + return to_pci_sysdata(bus)->nvme_remap_dev != NULL; +} + /* Can be used to override the logic in pci_scan_bus for skipping already-configured bus numbers - to be used for buggy BIOSes or architectures with incomplete PCI setup by the loader */ diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c index ddb798603201..7c20387d8202 100644 --- a/arch/x86/pci/common.c +++ b/arch/x86/pci/common.c @@ -723,12 +723,15 @@ int pci_ext_cfg_avail(void) return 0; } -#if IS_ENABLED(CONFIG_VMD) struct pci_dev *pci_real_dma_dev(struct pci_dev *dev) { +#if IS_ENABLED(CONFIG_VMD) if (is_vmd(dev->bus)) return to_pci_sysdata(dev->bus)->vmd_dev; +#endif + + if (is_nvme_remap(dev->bus)) + return to_pci_sysdata(dev->bus)->nvme_remap_dev; return dev; } -#endif diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched index 27f11320b8d1..e98585dd83e0 100644 --- a/block/Kconfig.iosched +++ b/block/Kconfig.iosched @@ -16,6 +16,20 @@ config MQ_IOSCHED_KYBER synchronous writes, it will self-tune queue depths to achieve that goal. +config MQ_IOSCHED_ADIOS + tristate "Adaptive Deadline I/O scheduler" + default m + help + The Adaptive Deadline I/O Scheduler (ADIOS) is a multi-queue I/O + scheduler with learning-based adaptive latency control. + +config MQ_IOSCHED_DEFAULT_ADIOS + bool "Enable ADIOS I/O scheduler as default MQ I/O scheduler" + depends on MQ_IOSCHED_ADIOS=y + default n + help + Enable the ADIOS I/O scheduler as the default scheduler for MQ I/O. + config IOSCHED_BFQ tristate "BFQ I/O scheduler" select BLK_ICQ diff --git a/block/Makefile b/block/Makefile index 3a941dc0d27f..94e3bf608bd2 100644 --- a/block/Makefile +++ b/block/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o obj-$(CONFIG_MQ_IOSCHED_DEADLINE) += mq-deadline.o obj-$(CONFIG_MQ_IOSCHED_KYBER) += kyber-iosched.o +obj-$(CONFIG_MQ_IOSCHED_ADIOS) += adios.o bfq-y := bfq-iosched.o bfq-wf2q.o bfq-cgroup.o obj-$(CONFIG_IOSCHED_BFQ) += bfq.o @@ -37,3 +38,10 @@ obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += blk-crypto.o blk-crypto-profile.o \ blk-crypto-sysfs.o obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o obj-$(CONFIG_BLOCK_HOLDER_DEPRECATED) += holder.o + +all: + make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules + +clean: + make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean + diff --git a/block/adios.c b/block/adios.c new file mode 100644 index 000000000000..f008d28cd059 --- /dev/null +++ b/block/adios.c @@ -0,0 +1,1344 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Adaptive Deadline I/O Scheduler (ADIOS) + * Copyright (C) 2025 Masahito Suzuki + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "elevator.h" +#include "blk.h" +#include "blk-mq.h" +#include "blk-mq-sched.h" + +#define ADIOS_VERSION "1.5.8" + +// Define operation types supported by ADIOS +enum adios_op_type { + ADIOS_READ = 0, + ADIOS_WRITE = 1, + ADIOS_DISCARD = 2, + ADIOS_OTHER = 3, + ADIOS_OPTYPES = 4, +}; + +// Global variable to control the latency +static u64 default_global_latency_window = 16000000ULL; +// Ratio below which batch queues should be refilled +static u8 default_bq_refill_below_ratio = 15; + +// Dynamic thresholds for shrinkage +static u32 default_lm_shrink_at_kreqs = 10000; +static u32 default_lm_shrink_at_gbytes = 100; +static u32 default_lm_shrink_resist = 2; + +// Latency targets for each operation type +static u64 default_latency_target[ADIOS_OPTYPES] = { + [ADIOS_READ] = 1ULL * NSEC_PER_MSEC, + [ADIOS_WRITE] = 2000ULL * NSEC_PER_MSEC, + [ADIOS_DISCARD] = 8000ULL * NSEC_PER_MSEC, + [ADIOS_OTHER] = 0ULL * NSEC_PER_MSEC, +}; + +// Maximum batch size limits for each operation type +static u32 default_batch_limit[ADIOS_OPTYPES] = { + [ADIOS_READ] = 24, + [ADIOS_WRITE] = 48, + [ADIOS_DISCARD] = 1, + [ADIOS_OTHER] = 1, +}; + +static u32 default_dl_prio[2] = {7, 0}; + +// Thresholds for latency model control +#define LM_BLOCK_SIZE_THRESHOLD 4096 +#define LM_SAMPLES_THRESHOLD 1024 +#define LM_INTERVAL_THRESHOLD 1500 +#define LM_OUTLIER_PERCENTILE 99 +#define LM_LAT_BUCKET_COUNT 64 + +// Structure to hold latency bucket data for small requests +struct latency_bucket_small { + u64 sum_latency; + u32 count; +}; + +// Structure to hold latency bucket data for large requests +struct latency_bucket_large { + u64 sum_latency; + u64 sum_block_size; + u32 count; +}; + +// Structure to hold the latency model context data +struct latency_model { + spinlock_t lock; + u64 base; + u64 slope; + u64 small_sum_delay; + u64 small_count; + u64 large_sum_delay; + u64 large_sum_bsize; + u64 last_update_jiffies; + + spinlock_t buckets_lock; + struct latency_bucket_small small_bucket[LM_LAT_BUCKET_COUNT]; + struct latency_bucket_large large_bucket[LM_LAT_BUCKET_COUNT]; + + u32 lm_shrink_at_kreqs; + u32 lm_shrink_at_gbytes; + u8 lm_shrink_resist; +}; + +#define ADIOS_BQ_PAGES 2 + +// Adios scheduler data +struct adios_data { + spinlock_t pq_lock; + struct list_head prio_queue; + + struct rb_root_cached dl_tree[2]; + spinlock_t lock; + u8 dl_queued; + s64 dl_bias; + s32 dl_prio[2]; + + u64 global_latency_window; + u64 latency_target[ADIOS_OPTYPES]; + u32 batch_limit[ADIOS_OPTYPES]; + u32 batch_actual_max_size[ADIOS_OPTYPES]; + u32 batch_actual_max_total; + u32 async_depth; + u8 bq_refill_below_ratio; + + u8 bq_page; + bool more_bq_ready; + struct list_head batch_queue[ADIOS_BQ_PAGES][ADIOS_OPTYPES]; + u32 batch_count[ADIOS_BQ_PAGES][ADIOS_OPTYPES]; + spinlock_t bq_lock; + + struct latency_model latency_model[ADIOS_OPTYPES]; + struct timer_list update_timer; + + atomic64_t total_pred_lat; + + struct kmem_cache *rq_data_pool; + struct kmem_cache *dl_group_pool; + + struct request_queue *queue; +}; + +// List of requests with the same deadline in the deadline-sorted tree +struct dl_group { + struct rb_node node; + struct list_head rqs; + u64 deadline; +} __attribute__((aligned(64))); + +// Structure to hold scheduler-specific data for each request +struct adios_rq_data { + struct list_head *dl_group; + struct list_head dl_node; + + struct request *rq; + u64 deadline; + u64 pred_lat; + u32 block_size; +} __attribute__((aligned(64))); + +static const int adios_prio_to_weight[40] = { + /* -20 */ 88761, 71755, 56483, 46273, 36291, + /* -15 */ 29154, 23254, 18705, 14949, 11916, + /* -10 */ 9548, 7620, 6100, 4904, 3906, + /* -5 */ 3121, 2501, 1991, 1586, 1277, + /* 0 */ 1024, 820, 655, 526, 423, + /* 5 */ 335, 272, 215, 172, 137, + /* 10 */ 110, 87, 70, 56, 45, + /* 15 */ 36, 29, 23, 18, 15, +}; + +// Count the number of entries in small buckets +static u32 lm_count_small_entries(struct latency_model *model) { + u32 total_count = 0; + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) + total_count += model->small_bucket[i].count; + return total_count; +} + +// Update the small buckets in the latency model +static bool lm_update_small_buckets(struct latency_model *model, + u32 total_count, bool count_all) { + u64 sum_latency = 0; + u32 sum_count = 0; + u32 cumulative_count = 0, threshold_count = 0; + u8 outlier_threshold_bucket = 0; + u8 outlier_percentile = LM_OUTLIER_PERCENTILE; + u8 reduction; + + if (count_all) + outlier_percentile = 100; + + // Calculate the threshold count for outlier detection + threshold_count = (total_count * outlier_percentile) / 100; + + // Identify the bucket that corresponds to the outlier threshold + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + cumulative_count += model->small_bucket[i].count; + if (cumulative_count >= threshold_count) { + outlier_threshold_bucket = i; + break; + } + } + + // Calculate the average latency, excluding outliers + for (u8 i = 0; i <= outlier_threshold_bucket; i++) { + struct latency_bucket_small *bucket = &model->small_bucket[i]; + if (i < outlier_threshold_bucket) { + sum_latency += bucket->sum_latency; + sum_count += bucket->count; + } else { + // The threshold bucket's contribution is proportional + u64 remaining_count = + threshold_count - (cumulative_count - bucket->count); + if (bucket->count > 0) { + sum_latency += + div_u64((bucket->sum_latency * remaining_count), bucket->count); + sum_count += remaining_count; + } + } + } + + // Shrink the model if it reaches at the readjustment threshold + if (model->small_count >= 1000ULL * model->lm_shrink_at_kreqs) { + reduction = model->lm_shrink_resist; + if (model->small_count >> reduction) { + model->small_sum_delay -= model->small_sum_delay >> reduction; + model->small_count -= model->small_count >> reduction; + } + } + + // Accumulate the average latency into the statistics + model->small_sum_delay += sum_latency; + model->small_count += sum_count; + + // Reset small bucket information + memset(model->small_bucket, 0, + sizeof(model->small_bucket[0]) * LM_LAT_BUCKET_COUNT); + + return true; +} + +// Count the number of entries in large buckets +static u32 lm_count_large_entries(struct latency_model *model) { + u32 total_count = 0; + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) + total_count += model->large_bucket[i].count; + return total_count; +} + +// Update the large buckets in the latency model +static bool lm_update_large_buckets( + struct latency_model *model, + u32 total_count, bool count_all) { + s64 sum_latency = 0; + u64 sum_block_size = 0, intercept; + u32 cumulative_count = 0, threshold_count = 0; + u8 outlier_threshold_bucket = 0; + u8 outlier_percentile = LM_OUTLIER_PERCENTILE; + u8 reduction; + + if (count_all) + outlier_percentile = 100; + + // Calculate the threshold count for outlier detection + threshold_count = (total_count * outlier_percentile) / 100; + + // Identify the bucket that corresponds to the outlier threshold + for (u8 i = 0; i < LM_LAT_BUCKET_COUNT; i++) { + cumulative_count += model->large_bucket[i].count; + if (cumulative_count >= threshold_count) { + outlier_threshold_bucket = i; + break; + } + } + + // Calculate the average latency and block size, excluding outliers + for (u8 i = 0; i <= outlier_threshold_bucket; i++) { + struct latency_bucket_large *bucket = &model->large_bucket[i]; + if (i < outlier_threshold_bucket) { + sum_latency += bucket->sum_latency; + sum_block_size += bucket->sum_block_size; + } else { + // The threshold bucket's contribution is proportional + u64 remaining_count = + threshold_count - (cumulative_count - bucket->count); + if (bucket->count > 0) { + sum_latency += + div_u64((bucket->sum_latency * remaining_count), bucket->count); + sum_block_size += + div_u64((bucket->sum_block_size * remaining_count), bucket->count); + } + } + } + + // Shrink the model if it reaches at the readjustment threshold + if (model->large_sum_bsize >= 0x40000000ULL * model->lm_shrink_at_gbytes) { + reduction = model->lm_shrink_resist; + if (model->large_sum_bsize >> reduction) { + model->large_sum_delay -= model->large_sum_delay >> reduction; + model->large_sum_bsize -= model->large_sum_bsize >> reduction; + } + } + + // Accumulate the average delay into the statistics + intercept = model->base * threshold_count; + if (sum_latency > intercept) + sum_latency -= intercept; + + model->large_sum_delay += sum_latency; + model->large_sum_bsize += sum_block_size; + + // Reset large bucket information + memset(model->large_bucket, 0, + sizeof(model->large_bucket[0]) * LM_LAT_BUCKET_COUNT); + + return true; +} + +// Update the latency model parameters and statistics +static void latency_model_update(struct latency_model *model) { + unsigned long flags; + u64 now; + u32 small_count, large_count; + bool time_elapsed; + bool small_processed = false, large_processed = false; + + guard(spinlock_irqsave)(&model->lock); + + spin_lock_irqsave(&model->buckets_lock, flags); + + // Whether enough time has elapsed since the last update + now = jiffies; + time_elapsed = unlikely(!model->base) || model->last_update_jiffies + + msecs_to_jiffies(LM_INTERVAL_THRESHOLD) <= now; + + // Count the number of entries in buckets + small_count = lm_count_small_entries(model); + large_count = lm_count_large_entries(model); + + // Update small buckets + if (small_count && (time_elapsed || + LM_SAMPLES_THRESHOLD <= small_count || !model->base)) + small_processed = lm_update_small_buckets( + model, small_count, !model->base); + // Update large buckets + if (large_count && (time_elapsed || + LM_SAMPLES_THRESHOLD <= large_count || !model->slope)) + large_processed = lm_update_large_buckets( + model, large_count, !model->slope); + + spin_unlock_irqrestore(&model->buckets_lock, flags); + + // Update the base parameter if small bucket was processed + if (small_processed && likely(model->small_count)) + model->base = div_u64(model->small_sum_delay, model->small_count); + + // Update the slope parameter if large bucket was processed + if (large_processed && likely(model->large_sum_bsize)) + model->slope = div_u64(model->large_sum_delay, + DIV_ROUND_UP_ULL(model->large_sum_bsize, 1024)); + + // Reset statistics and update last updated jiffies if time has elapsed + if (time_elapsed) + model->last_update_jiffies = now; +} + +// Determine the bucket index for a given measured and predicted latency +static u8 lm_input_bucket_index( + struct latency_model *model, u64 measured, u64 predicted) { + u8 bucket_index; + + if (measured < predicted * 2) + bucket_index = div_u64((measured * 20), predicted); + else if (measured < predicted * 5) + bucket_index = div_u64((measured * 10), predicted) + 20; + else + bucket_index = div_u64((measured * 3), predicted) + 40; + + return bucket_index; +} + +// Input latency data into the latency model +static void latency_model_input(struct latency_model *model, + u32 block_size, u64 latency, u64 pred_lat) { + unsigned long flags; + u8 bucket_index; + + spin_lock_irqsave(&model->buckets_lock, flags); + + if (block_size <= LM_BLOCK_SIZE_THRESHOLD) { + // Handle small requests + bucket_index = lm_input_bucket_index(model, latency, model->base ?: 1); + + if (bucket_index >= LM_LAT_BUCKET_COUNT) + bucket_index = LM_LAT_BUCKET_COUNT - 1; + + model->small_bucket[bucket_index].count++; + model->small_bucket[bucket_index].sum_latency += latency; + + if (unlikely(!model->base)) { + spin_unlock_irqrestore(&model->buckets_lock, flags); + latency_model_update(model); + return; + } + } else { + // Handle large requests + if (!model->base || !pred_lat) { + spin_unlock_irqrestore(&model->buckets_lock, flags); + return; + } + + bucket_index = lm_input_bucket_index(model, latency, pred_lat); + + if (bucket_index >= LM_LAT_BUCKET_COUNT) + bucket_index = LM_LAT_BUCKET_COUNT - 1; + + model->large_bucket[bucket_index].count++; + model->large_bucket[bucket_index].sum_latency += latency; + model->large_bucket[bucket_index].sum_block_size += block_size; + } + + spin_unlock_irqrestore(&model->buckets_lock, flags); +} + +// Predict the latency for a given block size using the latency model +static u64 latency_model_predict(struct latency_model *model, u32 block_size) { + u64 result; + + guard(spinlock_irqsave)(&model->lock); + // Predict latency based on the model + result = model->base; + if (block_size > LM_BLOCK_SIZE_THRESHOLD) + result += model->slope * + DIV_ROUND_UP_ULL(block_size - LM_BLOCK_SIZE_THRESHOLD, 1024); + + return result; +} + +// Determine the type of operation based on request flags +static u8 adios_optype(struct request *rq) { + switch (rq->cmd_flags & REQ_OP_MASK) { + case REQ_OP_READ: + return ADIOS_READ; + case REQ_OP_WRITE: + return ADIOS_WRITE; + case REQ_OP_DISCARD: + return ADIOS_DISCARD; + default: + return ADIOS_OTHER; + } +} + +static inline u8 adios_optype_not_read(struct request *rq) { + return (rq->cmd_flags & REQ_OP_MASK) != REQ_OP_READ; +} + +// Helper function to retrieve adios_rq_data from a request +static inline struct adios_rq_data *get_rq_data(struct request *rq) { + return rq->elv.priv[0]; +} + +// Add a request to the deadline-sorted red-black tree +static void add_to_dl_tree( + struct adios_data *ad, bool dl_idx, struct request *rq) { + struct rb_root_cached *root = &ad->dl_tree[dl_idx]; + struct rb_node **link = &(root->rb_root.rb_node), *parent = NULL; + bool leftmost = true; + struct adios_rq_data *rd = get_rq_data(rq); + struct dl_group *dlg; + + rd->block_size = blk_rq_bytes(rq); + u8 optype = adios_optype(rq); + rd->pred_lat = + latency_model_predict(&ad->latency_model[optype], rd->block_size); + rd->deadline = + rq->start_time_ns + ad->latency_target[optype] + rd->pred_lat; + + while (*link) { + dlg = rb_entry(*link, struct dl_group, node); + s64 diff = rd->deadline - dlg->deadline; + + parent = *link; + if (diff < 0) { + link = &((*link)->rb_left); + } else if (diff > 0) { + link = &((*link)->rb_right); + leftmost = false; + } else { // diff == 0 + goto found; + } + } + + dlg = rb_entry_safe(parent, struct dl_group, node); + if (!dlg || dlg->deadline != rd->deadline) { + dlg = kmem_cache_zalloc(ad->dl_group_pool, GFP_ATOMIC); + if (!dlg) + return; + dlg->deadline = rd->deadline; + INIT_LIST_HEAD(&dlg->rqs); + rb_link_node(&dlg->node, parent, link); + rb_insert_color_cached(&dlg->node, root, leftmost); + } +found: + list_add_tail(&rd->dl_node, &dlg->rqs); + rd->dl_group = &dlg->rqs; + ad->dl_queued |= 1 << dl_idx; +} + +// Remove a request from the deadline-sorted red-black tree +static void del_from_dl_tree( + struct adios_data *ad, bool dl_idx, struct request *rq) { + struct rb_root_cached *root = &ad->dl_tree[dl_idx]; + struct adios_rq_data *rd = get_rq_data(rq); + struct dl_group *dlg = container_of(rd->dl_group, struct dl_group, rqs); + + list_del_init(&rd->dl_node); + if (list_empty(&dlg->rqs)) { + rb_erase_cached(&dlg->node, root); + kmem_cache_free(ad->dl_group_pool, dlg); + } + rd->dl_group = NULL; + + if (RB_EMPTY_ROOT(&ad->dl_tree[dl_idx].rb_root)) + ad->dl_queued &= ~(1 << dl_idx); +} + +// Remove a request from the scheduler +static void remove_request(struct adios_data *ad, struct request *rq) { + bool dl_idx = adios_optype_not_read(rq); + struct request_queue *q = rq->q; + struct adios_rq_data *rd = get_rq_data(rq); + + list_del_init(&rq->queuelist); + + // We might not be on the rbtree, if we are doing an insert merge + if (rd->dl_group) + del_from_dl_tree(ad, dl_idx, rq); + + elv_rqhash_del(q, rq); + if (q->last_merge == rq) + q->last_merge = NULL; +} + +// Convert a queue depth to the corresponding word depth for shallow allocation +static int to_word_depth(struct blk_mq_hw_ctx *hctx, unsigned int qdepth) { + struct sbitmap_queue *bt = &hctx->sched_tags->bitmap_tags; + const unsigned int nrr = hctx->queue->nr_requests; + + return ((qdepth << bt->sb.shift) + nrr - 1) / nrr; +} + +// Limit the depth of request allocation for asynchronous and write requests +static void adios_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { + struct adios_data *ad = data->q->elevator->elevator_data; + + // Do not throttle synchronous reads + if (op_is_sync(opf) && !op_is_write(opf)) + return; + + data->shallow_depth = to_word_depth(data->hctx, ad->async_depth); +} + +// Update async_depth when the number of requests in the queue changes +static void adios_depth_updated(struct blk_mq_hw_ctx *hctx) { + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + struct blk_mq_tags *tags = hctx->sched_tags; + + ad->async_depth = q->nr_requests; + + sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, 1); +} + +// Handle request merging after a merge operation +static void adios_request_merged(struct request_queue *q, struct request *req, + enum elv_merge type) { + bool dl_idx = adios_optype_not_read(req); + struct adios_data *ad = q->elevator->elevator_data; + + // if the merge was a front merge, we need to reposition request + if (type == ELEVATOR_FRONT_MERGE) { + del_from_dl_tree(ad, dl_idx, req); + add_to_dl_tree(ad, dl_idx, req); + } +} + +// Handle merging of requests after one has been merged into another +static void adios_merged_requests(struct request_queue *q, struct request *req, + struct request *next) { + struct adios_data *ad = q->elevator->elevator_data; + + lockdep_assert_held(&ad->lock); + + // kill knowledge of next, this one is a goner + remove_request(ad, next); +} + +// Try to merge a bio into an existing rq before associating it with an rq +static bool adios_bio_merge(struct request_queue *q, struct bio *bio, + unsigned int nr_segs) { + unsigned long flags; + struct adios_data *ad = q->elevator->elevator_data; + struct request *free = NULL; + bool ret; + + spin_lock_irqsave(&ad->lock, flags); + ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); + spin_unlock_irqrestore(&ad->lock, flags); + + if (free) + blk_mq_free_request(free); + + return ret; +} + +// Insert a request into the scheduler +static void insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + blk_insert_t insert_flags, struct list_head *free) { + unsigned long flags; + bool dl_idx = adios_optype_not_read(rq); + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + + lockdep_assert_held(&ad->lock); + + if (insert_flags & BLK_MQ_INSERT_AT_HEAD) { + spin_lock_irqsave(&ad->pq_lock, flags); + list_add(&rq->queuelist, &ad->prio_queue); + spin_unlock_irqrestore(&ad->pq_lock, flags); + return; + } + + if (blk_mq_sched_try_insert_merge(q, rq, free)) + return; + + add_to_dl_tree(ad, dl_idx, rq); + + if (rq_mergeable(rq)) { + elv_rqhash_add(q, rq); + if (!q->last_merge) + q->last_merge = rq; + } +} + +// Insert multiple requests into the scheduler +static void adios_insert_requests(struct blk_mq_hw_ctx *hctx, + struct list_head *list, + blk_insert_t insert_flags) { + unsigned long flags; + struct request_queue *q = hctx->queue; + struct adios_data *ad = q->elevator->elevator_data; + LIST_HEAD(free); + + spin_lock_irqsave(&ad->lock, flags); + while (!list_empty(list)) { + struct request *rq; + + rq = list_first_entry(list, struct request, queuelist); + list_del_init(&rq->queuelist); + insert_request(hctx, rq, insert_flags, &free); + } + spin_unlock_irqrestore(&ad->lock, flags); + + blk_mq_free_requests(&free); +} + +// Prepare a request before it is inserted into the scheduler +static void adios_prepare_request(struct request *rq) { + struct adios_data *ad = rq->q->elevator->elevator_data; + struct adios_rq_data *rd; + + rq->elv.priv[0] = NULL; + + /* Allocate adios_rq_data from the memory pool */ + rd = kmem_cache_zalloc(ad->rq_data_pool, GFP_ATOMIC); + if (WARN(!rd, "adios_prepare_request: " + "Failed to allocate memory from rq_data_pool. rd is NULL\n")) + return; + + rd->rq = rq; + rq->elv.priv[0] = rd; +} + +static struct adios_rq_data *get_dl_first_rd(struct adios_data *ad, bool idx) { + struct rb_root_cached *root = &ad->dl_tree[idx]; + struct rb_node *first = rb_first_cached(root); + struct dl_group *dl_group = rb_entry(first, struct dl_group, node); + + return list_first_entry(&dl_group->rqs, struct adios_rq_data, dl_node); +} + +// Select the next request to dispatch from the deadline-sorted red-black tree +static struct request *next_request(struct adios_data *ad) { + struct adios_rq_data *rd; + bool dl_idx, bias_idx, reduce_bias; + + if (!ad->dl_queued) + return NULL; + + dl_idx = ad->dl_queued >> 1; + rd = get_dl_first_rd(ad, dl_idx); + + bias_idx = ad->dl_bias < 0; + reduce_bias = (bias_idx == dl_idx); + + if (ad->dl_queued == 0x3) { + struct adios_rq_data *trd[2]; + trd[0] = get_dl_first_rd(ad, 0); + trd[1] = rd; + + rd = trd[bias_idx]; + + reduce_bias = + (trd[bias_idx]->deadline > trd[((u8)bias_idx + 1) % 2]->deadline); + } + + if (reduce_bias) { + s64 sign = ((int)bias_idx << 1) - 1; + if (unlikely(!rd->pred_lat)) + ad->dl_bias = sign; + else { + ad->dl_bias += sign * (s64)((rd->pred_lat * + adios_prio_to_weight[ad->dl_prio[bias_idx] + 20]) >> 10); + } + } + + return rd->rq; +} + +// Reset the batch queue counts for a given page +static void reset_batch_counts(struct adios_data *ad, u8 page) { + memset(&ad->batch_count[page], 0, sizeof(ad->batch_count[page])); +} + +// Initialize all batch queues +static void init_batch_queues(struct adios_data *ad) { + for (u8 page = 0; page < ADIOS_BQ_PAGES; page++) { + reset_batch_counts(ad, page); + + for (u8 optype = 0; optype < ADIOS_OPTYPES; optype++) + INIT_LIST_HEAD(&ad->batch_queue[page][optype]); + } +} + +// Fill the batch queues with requests from the deadline-sorted red-black tree +static bool fill_batch_queues(struct adios_data *ad, u64 current_lat) { + unsigned long flags; + u32 count = 0; + u32 optype_count[ADIOS_OPTYPES] = {0}; + u8 page = (ad->bq_page + 1) % ADIOS_BQ_PAGES; + + reset_batch_counts(ad, page); + + spin_lock_irqsave(&ad->lock, flags); + while (true) { + struct request *rq = next_request(ad); + if (!rq) + break; + + struct adios_rq_data *rd = get_rq_data(rq); + u8 optype = adios_optype(rq); + current_lat += rd->pred_lat; + + // Check batch size and total predicted latency + if (count && (!ad->latency_model[optype].base || + ad->batch_count[page][optype] >= ad->batch_limit[optype] || + current_lat > ad->global_latency_window)) { + break; + } + + remove_request(ad, rq); + + // Add request to the corresponding batch queue + list_add_tail(&rq->queuelist, &ad->batch_queue[page][optype]); + ad->batch_count[page][optype]++; + atomic64_add(rd->pred_lat, &ad->total_pred_lat); + optype_count[optype]++; + count++; + } + spin_unlock_irqrestore(&ad->lock, flags); + + if (count) { + ad->more_bq_ready = true; + for (u8 optype = 0; optype < ADIOS_OPTYPES; optype++) { + if (ad->batch_actual_max_size[optype] < optype_count[optype]) + ad->batch_actual_max_size[optype] = optype_count[optype]; + } + if (ad->batch_actual_max_total < count) + ad->batch_actual_max_total = count; + } + return count; +} + +// Flip to the next batch queue page +static void flip_bq_page(struct adios_data *ad) { + ad->more_bq_ready = false; + ad->bq_page = (ad->bq_page + 1) % ADIOS_BQ_PAGES; +} + +// Dispatch a request from the batch queues +static struct request *dispatch_from_bq(struct adios_data *ad) { + struct request *rq = NULL; + u64 tpl; + + guard(spinlock_irqsave)(&ad->bq_lock); + + tpl = atomic64_read(&ad->total_pred_lat); + + if (!ad->more_bq_ready && (!tpl || + tpl < div_u64(ad->global_latency_window * ad->bq_refill_below_ratio, 100) )) + fill_batch_queues(ad, tpl); + +again: + // Check if there are any requests in the batch queues + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + if (!list_empty(&ad->batch_queue[ad->bq_page][i])) { + rq = list_first_entry(&ad->batch_queue[ad->bq_page][i], + struct request, queuelist); + list_del_init(&rq->queuelist); + return rq; + } + } + + // If there's more batch queue page available, flip to it and retry + if (ad->more_bq_ready) { + flip_bq_page(ad); + goto again; + } + + return NULL; +} + +// Dispatch a request from the priority queue +static struct request *dispatch_from_pq(struct adios_data *ad) { + struct request *rq = NULL; + + guard(spinlock_irqsave)(&ad->pq_lock); + + if (!list_empty(&ad->prio_queue)) { + rq = list_first_entry(&ad->prio_queue, struct request, queuelist); + list_del_init(&rq->queuelist); + } + return rq; +} + +// Dispatch a request to the hardware queue +static struct request *adios_dispatch_request(struct blk_mq_hw_ctx *hctx) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + struct request *rq; + + rq = dispatch_from_pq(ad); + if (rq) goto found; + rq = dispatch_from_bq(ad); + if (!rq) return NULL; +found: + rq->rq_flags |= RQF_STARTED; + return rq; +} + +// Timer callback function to periodically update latency models +static void update_timer_callback(struct timer_list *t) { + struct adios_data *ad = from_timer(ad, t, update_timer); + + for (u8 optype = 0; optype < ADIOS_OPTYPES; optype++) + latency_model_update(&ad->latency_model[optype]); +} + +// Handle the completion of a request +static void adios_completed_request(struct request *rq, u64 now) { + struct adios_data *ad = rq->q->elevator->elevator_data; + struct adios_rq_data *rd = get_rq_data(rq); + + atomic64_sub(rd->pred_lat, &ad->total_pred_lat); + + if (!rq->io_start_time_ns || !rd->block_size) + return; + u64 latency = now - rq->io_start_time_ns; + u8 optype = adios_optype(rq); + latency_model_input(&ad->latency_model[optype], + rd->block_size, latency, rd->pred_lat); + timer_reduce(&ad->update_timer, jiffies + msecs_to_jiffies(100)); +} + +// Clean up after a request is finished +static void adios_finish_request(struct request *rq) { + struct adios_data *ad = rq->q->elevator->elevator_data; + + if (rq->elv.priv[0]) { + // Free adios_rq_data back to the memory pool + kmem_cache_free(ad->rq_data_pool, get_rq_data(rq)); + rq->elv.priv[0] = NULL; + } +} + +static inline bool pq_has_work(struct adios_data *ad) { + guard(spinlock_irqsave)(&ad->pq_lock); + return !list_empty(&ad->prio_queue); +} + +static inline bool bq_has_work(struct adios_data *ad) { + guard(spinlock_irqsave)(&ad->bq_lock); + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) + if (!list_empty(&ad->batch_queue[ad->bq_page][i])) + return true; + + return ad->more_bq_ready; +} + +static inline bool dl_tree_has_work(struct adios_data *ad) { + guard(spinlock_irqsave)(&ad->lock); + return ad->dl_queued; +} + +// Check if there are any requests available for dispatch +static bool adios_has_work(struct blk_mq_hw_ctx *hctx) { + struct adios_data *ad = hctx->queue->elevator->elevator_data; + + return pq_has_work(ad) || bq_has_work(ad) || dl_tree_has_work(ad); +} + +// Initialize the scheduler-specific data for a hardware queue +static int adios_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { + adios_depth_updated(hctx); + return 0; +} + +// Initialize the scheduler-specific data when initializing the request queue +static int adios_init_sched(struct request_queue *q, struct elevator_type *e) { + struct adios_data *ad; + struct elevator_queue *eq; + int ret = -ENOMEM; + + eq = elevator_alloc(q, e); + if (!eq) + return ret; + + ad = kzalloc_node(sizeof(*ad), GFP_KERNEL, q->node); + if (!ad) + goto put_eq; + + // Create a memory pool for adios_rq_data + ad->rq_data_pool = kmem_cache_create("rq_data_pool", + sizeof(struct adios_rq_data), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ad->rq_data_pool) { + pr_err("adios: Failed to create rq_data_pool\n"); + goto free_ad; + } + + /* Create a memory pool for dl_group */ + ad->dl_group_pool = kmem_cache_create("dl_group_pool", + sizeof(struct dl_group), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ad->dl_group_pool) { + pr_err("adios: Failed to create dl_group_pool\n"); + goto destroy_rq_data_pool; + } + + eq->elevator_data = ad; + + ad->global_latency_window = default_global_latency_window; + ad->bq_refill_below_ratio = default_bq_refill_below_ratio; + + INIT_LIST_HEAD(&ad->prio_queue); + for (u8 i = 0; i < 2; i++) + ad->dl_tree[i] = RB_ROOT_CACHED; + ad->dl_bias = 0; + ad->dl_queued = 0x0; + for (u8 i = 0; i < 2; i++) + ad->dl_prio[i] = default_dl_prio[i]; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + struct latency_model *model = &ad->latency_model[i]; + spin_lock_init(&model->lock); + spin_lock_init(&model->buckets_lock); + memset(model->small_bucket, 0, + sizeof(model->small_bucket[0]) * LM_LAT_BUCKET_COUNT); + memset(model->large_bucket, 0, + sizeof(model->large_bucket[0]) * LM_LAT_BUCKET_COUNT); + model->last_update_jiffies = jiffies; + model->lm_shrink_at_kreqs = default_lm_shrink_at_kreqs; + model->lm_shrink_at_gbytes = default_lm_shrink_at_gbytes; + model->lm_shrink_resist = default_lm_shrink_resist; + + ad->latency_target[i] = default_latency_target[i]; + ad->batch_limit[i] = default_batch_limit[i]; + } + timer_setup(&ad->update_timer, update_timer_callback, 0); + init_batch_queues(ad); + + spin_lock_init(&ad->lock); + spin_lock_init(&ad->pq_lock); + spin_lock_init(&ad->bq_lock); + + /* We dispatch from request queue wide instead of hw queue */ + blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q); + + ad->queue = q; + blk_stat_enable_accounting(q); + + q->elevator = eq; + return 0; + +destroy_rq_data_pool: + kmem_cache_destroy(ad->rq_data_pool); +free_ad: + kfree(ad); +put_eq: + kobject_put(&eq->kobj); + return ret; +} + +// Clean up and free resources when exiting the scheduler +static void adios_exit_sched(struct elevator_queue *e) { + struct adios_data *ad = e->elevator_data; + + timer_shutdown_sync(&ad->update_timer); + + WARN_ON_ONCE(!list_empty(&ad->prio_queue)); + + if (ad->rq_data_pool) + kmem_cache_destroy(ad->rq_data_pool); + + if (ad->dl_group_pool) + kmem_cache_destroy(ad->dl_group_pool); + + blk_stat_disable_accounting(ad->queue); + + kfree(ad); +} + +// Define sysfs attributes for read operation latency model +#define SYSFS_OPTYPE_DECL(name, optype) \ +static ssize_t adios_lat_model_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + struct latency_model *model = &ad->latency_model[optype]; \ + ssize_t len = 0; \ + guard(spinlock_irqsave)(&model->lock); \ + len += sprintf(page, "base : %llu ns\n", model->base); \ + len += sprintf(page + len, "slope: %llu ns/KiB\n", model->slope);\ + return len; \ +} \ +static ssize_t adios_lat_target_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long nsec; \ + int ret; \ + ret = kstrtoul(page, 10, &nsec); \ + if (ret) \ + return ret; \ + ad->latency_model[optype].base = 0ULL; \ + ad->latency_target[optype] = nsec; \ + return count; \ +} \ +static ssize_t adios_lat_target_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%llu\n", ad->latency_target[optype]); \ +} \ +static ssize_t adios_batch_limit_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + unsigned long max_batch; \ + int ret; \ + ret = kstrtoul(page, 10, &max_batch); \ + if (ret || max_batch == 0) \ + return -EINVAL; \ + struct adios_data *ad = e->elevator_data; \ + ad->batch_limit[optype] = max_batch; \ + return count; \ +} \ +static ssize_t adios_batch_limit_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + return sprintf(page, "%u\n", ad->batch_limit[optype]); \ +} + +SYSFS_OPTYPE_DECL(read, ADIOS_READ); +SYSFS_OPTYPE_DECL(write, ADIOS_WRITE); +SYSFS_OPTYPE_DECL(discard, ADIOS_DISCARD); + +// Show the maximum batch size actually achieved for each operation type +static ssize_t adios_batch_actual_max_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + u32 total_count, read_count, write_count, discard_count; + + total_count = ad->batch_actual_max_total; + read_count = ad->batch_actual_max_size[ADIOS_READ]; + write_count = ad->batch_actual_max_size[ADIOS_WRITE]; + discard_count = ad->batch_actual_max_size[ADIOS_DISCARD]; + + return sprintf(page, + "Total : %u\nDiscard: %u\nRead : %u\nWrite : %u\n", + total_count, discard_count, read_count, write_count); +} + +// Set the global latency window +static ssize_t adios_global_latency_window_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + unsigned long nsec; + int ret; + + ret = kstrtoul(page, 10, &nsec); + if (ret) + return ret; + + ad->global_latency_window = nsec; + + return count; +} + +// Show the global latency window +static ssize_t adios_global_latency_window_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + return sprintf(page, "%llu\n", ad->global_latency_window); +} + +// Show the bq_refill_below_ratio +static ssize_t adios_bq_refill_below_ratio_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + return sprintf(page, "%d\n", ad->bq_refill_below_ratio); +} + +// Set the bq_refill_below_ratio +static ssize_t adios_bq_refill_below_ratio_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + int ratio; + int ret; + + ret = kstrtoint(page, 10, &ratio); + if (ret || ratio < 0 || ratio > 100) + return -EINVAL; + + ad->bq_refill_below_ratio = ratio; + + return count; +} + +// Show the read priority +static ssize_t adios_read_priority_show( + struct elevator_queue *e, char *page) { + struct adios_data *ad = e->elevator_data; + return sprintf(page, "%d\n", ad->dl_prio[0]); +} + +// Set the read priority +static ssize_t adios_read_priority_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + int prio; + int ret; + + ret = kstrtoint(page, 10, &prio); + if (ret || prio < -20 || prio > 19) + return -EINVAL; + + guard(spinlock_irqsave)(&ad->lock); + ad->dl_prio[0] = prio; + ad->dl_bias = 0; + + return count; +} + +// Reset batch queue statistics +static ssize_t adios_reset_bq_stats_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + unsigned long val; + int ret; + + ret = kstrtoul(page, 10, &val); + if (ret || val != 1) + return -EINVAL; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) + ad->batch_actual_max_size[i] = 0; + + ad->batch_actual_max_total = 0; + + return count; +} + +// Reset the latency model parameters +static ssize_t adios_reset_lat_model_store( + struct elevator_queue *e, const char *page, size_t count) { + struct adios_data *ad = e->elevator_data; + unsigned long val; + int ret; + + ret = kstrtoul(page, 10, &val); + if (ret || val != 1) + return -EINVAL; + + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { + struct latency_model *model = &ad->latency_model[i]; + unsigned long flags; + spin_lock_irqsave(&model->lock, flags); + model->base = 0ULL; + model->slope = 0ULL; + model->small_sum_delay = 0ULL; + model->small_count = 0ULL; + model->large_sum_delay = 0ULL; + model->large_sum_bsize = 0ULL; + spin_unlock_irqrestore(&model->lock, flags); + } + + return count; +} + +// Show the ADIOS version +static ssize_t adios_version_show(struct elevator_queue *e, char *page) { + return sprintf(page, "%s\n", ADIOS_VERSION); +} + +// Define sysfs attributes for dynamic thresholds +#define SHRINK_THRESHOLD_ATTR_RW(name, model_field, min_value, max_value) \ +static ssize_t adios_shrink_##name##_store( \ + struct elevator_queue *e, const char *page, size_t count) { \ + struct adios_data *ad = e->elevator_data; \ + unsigned long val; \ + int ret; \ + ret = kstrtoul(page, 10, &val); \ + if (ret || val < min_value || val > max_value) \ + return -EINVAL; \ + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { \ + struct latency_model *model = &ad->latency_model[i]; \ + unsigned long flags; \ + spin_lock_irqsave(&model->lock, flags); \ + model->model_field = val; \ + spin_unlock_irqrestore(&model->lock, flags); \ + } \ + return count; \ +} \ +static ssize_t adios_shrink_##name##_show( \ + struct elevator_queue *e, char *page) { \ + struct adios_data *ad = e->elevator_data; \ + u32 val = 0; \ + for (u8 i = 0; i < ADIOS_OPTYPES; i++) { \ + struct latency_model *model = &ad->latency_model[i]; \ + unsigned long flags; \ + spin_lock_irqsave(&model->lock, flags); \ + val = model->model_field; \ + spin_unlock_irqrestore(&model->lock, flags); \ + } \ + return sprintf(page, "%u\n", val); \ +} + +SHRINK_THRESHOLD_ATTR_RW(at_kreqs, lm_shrink_at_kreqs, 1, 100000) +SHRINK_THRESHOLD_ATTR_RW(at_gbytes, lm_shrink_at_gbytes, 1, 1000) +SHRINK_THRESHOLD_ATTR_RW(resist, lm_shrink_resist, 1, 3) + +// Define sysfs attributes +#define AD_ATTR(name, show_func, store_func) \ + __ATTR(name, 0644, show_func, store_func) +#define AD_ATTR_RW(name) \ + __ATTR(name, 0644, adios_##name##_show, adios_##name##_store) +#define AD_ATTR_RO(name) \ + __ATTR(name, 0644, adios_##name##_show, NULL) +#define AD_ATTR_WO(name) \ + __ATTR(name, 0644, NULL, adios_##name##_store) + +// Define sysfs attributes for ADIOS scheduler +static struct elv_fs_entry adios_sched_attrs[] = { + AD_ATTR_RO(batch_actual_max), + AD_ATTR_RW(bq_refill_below_ratio), + AD_ATTR_RW(global_latency_window), + + AD_ATTR_RW(batch_limit_read), + AD_ATTR_RW(batch_limit_write), + AD_ATTR_RW(batch_limit_discard), + + AD_ATTR_RO(lat_model_read), + AD_ATTR_RO(lat_model_write), + AD_ATTR_RO(lat_model_discard), + + AD_ATTR_RW(lat_target_read), + AD_ATTR_RW(lat_target_write), + AD_ATTR_RW(lat_target_discard), + + AD_ATTR_RW(shrink_at_kreqs), + AD_ATTR_RW(shrink_at_gbytes), + AD_ATTR_RW(shrink_resist), + + AD_ATTR_RW(read_priority), + + AD_ATTR_WO(reset_bq_stats), + AD_ATTR_WO(reset_lat_model), + AD_ATTR(adios_version, adios_version_show, NULL), + + __ATTR_NULL +}; + +// Define the ADIOS scheduler type +static struct elevator_type mq_adios = { + .ops = { + .next_request = elv_rb_latter_request, + .former_request = elv_rb_former_request, + .limit_depth = adios_limit_depth, + .depth_updated = adios_depth_updated, + .request_merged = adios_request_merged, + .requests_merged = adios_merged_requests, + .bio_merge = adios_bio_merge, + .insert_requests = adios_insert_requests, + .prepare_request = adios_prepare_request, + .dispatch_request = adios_dispatch_request, + .completed_request = adios_completed_request, + .finish_request = adios_finish_request, + .has_work = adios_has_work, + .init_hctx = adios_init_hctx, + .init_sched = adios_init_sched, + .exit_sched = adios_exit_sched, + }, + .elevator_attrs = adios_sched_attrs, + .elevator_name = "adios", + .elevator_owner = THIS_MODULE, +}; +MODULE_ALIAS("mq-adios-iosched"); + +#define ADIOS_PROGNAME "Adaptive Deadline I/O Scheduler" +#define ADIOS_AUTHOR "Masahito Suzuki" + +// Initialize the ADIOS scheduler module +static int __init adios_init(void) { + printk(KERN_INFO "%s %s by %s\n", + ADIOS_PROGNAME, ADIOS_VERSION, ADIOS_AUTHOR); + return elv_register(&mq_adios); +} + +// Exit the ADIOS scheduler module +static void __exit adios_exit(void) { + elv_unregister(&mq_adios); +} + +module_init(adios_init); +module_exit(adios_exit); + +MODULE_AUTHOR(ADIOS_AUTHOR); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION(ADIOS_PROGNAME); \ No newline at end of file diff --git a/block/elevator.c b/block/elevator.c index b4d08026b02c..d0ab14375925 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -556,11 +556,23 @@ static struct elevator_type *elevator_get_default(struct request_queue *q) if (q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT) return NULL; +#ifdef CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + return elevator_find_get("adios"); +#endif // CONFIG_MQ_IOSCHED_DEFAULT_ADIOS + if (q->nr_hw_queues != 1 && !blk_mq_is_shared_tags(q->tag_set->flags)) +#if defined(CONFIG_CACHY) + return elevator_find_get("mq-deadline"); +#else return NULL; +#endif +#if defined(CONFIG_CACHY) && defined(CONFIG_IOSCHED_BFQ) + return elevator_find_get("bfq"); +#else return elevator_find_get("mq-deadline"); +#endif } /* diff --git a/drivers/Makefile b/drivers/Makefile index b5749cf67044..5beba9f57254 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -64,14 +64,8 @@ obj-y += char/ # iommu/ comes before gpu as gpu are using iommu controllers obj-y += iommu/ -# gpu/ comes after char for AGP vs DRM startup and after iommu -obj-y += gpu/ - obj-$(CONFIG_CONNECTOR) += connector/ -# i810fb depends on char/agp/ -obj-$(CONFIG_FB_I810) += video/fbdev/i810/ - obj-$(CONFIG_PARPORT) += parport/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ @@ -83,6 +77,13 @@ obj-y += macintosh/ obj-y += scsi/ obj-y += nvme/ obj-$(CONFIG_ATA) += ata/ + +# gpu/ comes after char for AGP vs DRM startup and after iommu +obj-y += gpu/ + +# i810fb depends on char/agp/ +obj-$(CONFIG_FB_I810) += video/fbdev/i810/ + obj-$(CONFIG_TARGET_CORE) += target/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 163ac909bd06..83a1e8b5443c 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1629,7 +1629,7 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance) } #endif -static void ahci_remap_check(struct pci_dev *pdev, int bar, +static int ahci_remap_check(struct pci_dev *pdev, int bar, struct ahci_host_priv *hpriv) { int i; @@ -1642,7 +1642,7 @@ static void ahci_remap_check(struct pci_dev *pdev, int bar, pci_resource_len(pdev, bar) < SZ_512K || bar != AHCI_PCI_BAR_STANDARD || !(readl(hpriv->mmio + AHCI_VSCAP) & 1)) - return; + return 0; cap = readq(hpriv->mmio + AHCI_REMAP_CAP); for (i = 0; i < AHCI_MAX_REMAP; i++) { @@ -1657,18 +1657,11 @@ static void ahci_remap_check(struct pci_dev *pdev, int bar, } if (!hpriv->remapped_nvme) - return; - - dev_warn(&pdev->dev, "Found %u remapped NVMe devices.\n", - hpriv->remapped_nvme); - dev_warn(&pdev->dev, - "Switch your BIOS from RAID to AHCI mode to use them.\n"); + return 0; - /* - * Don't rely on the msi-x capability in the remap case, - * share the legacy interrupt across ahci and remapped devices. - */ - hpriv->flags |= AHCI_HFLAG_NO_MSI; + /* Abort probe, allowing intel-nvme-remap to step in when available */ + dev_info(&pdev->dev, "Device will be handled by intel-nvme-remap.\n"); + return -ENODEV; } static int ahci_get_irq_vector(struct ata_host *host, int port) @@ -1912,7 +1905,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; /* detect remapped nvme devices */ - ahci_remap_check(pdev, ahci_pci_bar, hpriv); + rc = ahci_remap_check(pdev, ahci_pci_bar, hpriv); + if (rc) + return rc; sysfs_add_file_to_group(&pdev->dev.kobj, &dev_attr_remapped_nvme.attr, diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 2c5c228408bf..918e2bebfe78 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -9,7 +9,6 @@ config X86_INTEL_PSTATE select ACPI_PROCESSOR if ACPI select ACPI_CPPC_LIB if X86_64 && ACPI && SCHED_MC_PRIO select CPU_FREQ_GOV_PERFORMANCE - select CPU_FREQ_GOV_SCHEDUTIL if SMP help This driver provides a P state for Intel core processors. The driver implements an internal governor and will become @@ -39,7 +38,6 @@ config X86_AMD_PSTATE depends on X86 && ACPI select ACPI_PROCESSOR select ACPI_CPPC_LIB if X86_64 - select CPU_FREQ_GOV_SCHEDUTIL if SMP help This driver adds a CPUFreq driver which utilizes a fine grain processor performance frequency control range instead of legacy diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ba9bf06f1c77..b8ceac594acd 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3828,6 +3828,8 @@ static int __init intel_pstate_setup(char *str) if (!strcmp(str, "disable")) no_load = 1; + else if (!strcmp(str, "enable")) + no_load = 0; else if (!strcmp(str, "active")) default_driver = &intel_pstate; else if (!strcmp(str, "passive")) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index c3641331d4de..ce1072fe4921 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -161,6 +161,7 @@ struct amdgpu_watchdog_timer { */ extern int amdgpu_modeset; extern unsigned int amdgpu_vram_limit; +extern int amdgpu_ignore_min_pcap; extern int amdgpu_vis_vram_limit; extern int amdgpu_gart_size; extern int amdgpu_gtt_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 72c807f5822e..6ef3fedc1233 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -143,6 +143,7 @@ enum AMDGPU_DEBUG_MASK { }; unsigned int amdgpu_vram_limit = UINT_MAX; +int amdgpu_ignore_min_pcap = 0; /* do not ignore by default */ int amdgpu_vis_vram_limit; int amdgpu_gart_size = -1; /* auto */ int amdgpu_gtt_size = -1; /* auto */ @@ -262,6 +263,15 @@ struct amdgpu_watchdog_timer amdgpu_watchdog_timer = { .period = 0x0, /* default to 0x0 (timeout disable) */ }; +/** + * DOC: ignore_min_pcap (int) + * Ignore the minimum power cap. + * Useful on graphics cards where the minimum power cap is very high. + * The default is 0 (Do not ignore). + */ +MODULE_PARM_DESC(ignore_min_pcap, "Ignore the minimum power cap"); +module_param_named(ignore_min_pcap, amdgpu_ignore_min_pcap, int, 0600); + /** * DOC: vramlimit (int) * Restrict the total amount of VRAM in MiB for testing. The default is 0 (Use full VRAM). diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index abd3b6564373..46937e6fa78d 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -56,4 +56,10 @@ config DRM_AMD_SECURE_DISPLAY This option enables the calculation of crc of specific region via debugfs. Cooperate with specific DMCU FW. +config AMD_PRIVATE_COLOR + bool "Enable KMS color management by AMD for AMD" + default n + help + This option extends the KMS color management API with AMD driver-specific properties to enhance the color management support on AMD Steam Deck. + endmenu diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index a187cdb43e7e..ef0f71ce25d5 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -4726,7 +4726,7 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) return r; } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR if (amdgpu_dm_create_color_properties(adev)) { dc_state_release(state->context); kfree(state); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index ebabfe3a512f..4d3ebcaacca1 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -97,7 +97,7 @@ static inline struct fixed31_32 amdgpu_dm_fixpt_from_s3132(__u64 x) return val; } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR /* Pre-defined Transfer Functions (TF) * * AMD driver supports pre-defined mathematical functions for transferring diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index e8bdd7f0c460..b4aafe6103bc 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -481,7 +481,7 @@ static int amdgpu_dm_crtc_late_register(struct drm_crtc *crtc) } #endif -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR /** * dm_crtc_additional_color_mgmt - enable additional color properties * @crtc: DRM CRTC @@ -563,7 +563,7 @@ static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = { #if defined(CONFIG_DEBUG_FS) .late_register = amdgpu_dm_crtc_late_register, #endif -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR .atomic_set_property = amdgpu_dm_atomic_crtc_set_property, .atomic_get_property = amdgpu_dm_atomic_crtc_get_property, #endif @@ -742,7 +742,7 @@ int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm, drm_mode_crtc_set_gamma_size(&acrtc->base, MAX_COLOR_LEGACY_LUT_ENTRIES); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR dm_crtc_additional_color_mgmt(&acrtc->base); #endif return 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index 3e0f45f1711c..f645cb7831a0 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1601,7 +1601,7 @@ static void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane, drm_atomic_helper_plane_destroy_state(plane, state); } -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR static void dm_atomic_plane_attach_color_mgmt_properties(struct amdgpu_display_manager *dm, struct drm_plane *plane) @@ -1792,7 +1792,7 @@ static const struct drm_plane_funcs dm_plane_funcs = { .atomic_duplicate_state = amdgpu_dm_plane_drm_plane_duplicate_state, .atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state, .format_mod_supported = amdgpu_dm_plane_format_mod_supported, -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR .atomic_set_property = dm_atomic_plane_set_property, .atomic_get_property = dm_atomic_plane_get_property, #endif @@ -1888,7 +1888,7 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm, else drm_plane_helper_add(plane, &dm_plane_helper_funcs); -#ifdef AMD_PRIVATE_COLOR +#ifdef CONFIG_AMD_PRIVATE_COLOR dm_atomic_plane_attach_color_mgmt_properties(dm, plane); #endif /* Create (reset) the plane state */ diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 922def51685b..da76611edb10 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -3055,6 +3055,9 @@ static ssize_t amdgpu_hwmon_show_power_cap_min(struct device *dev, struct device_attribute *attr, char *buf) { + if (amdgpu_ignore_min_pcap) + return sysfs_emit(buf, "%i\n", 0); + return amdgpu_hwmon_show_power_cap_generic(dev, attr, buf, PP_PWR_LIMIT_MIN); } diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 46cce1d2aaf3..e3adb7aea969 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -2854,7 +2854,10 @@ int smu_get_power_limit(void *handle, *limit = smu->max_power_limit; break; case SMU_PPT_LIMIT_MIN: - *limit = smu->min_power_limit; + if (amdgpu_ignore_min_pcap) + *limit = 0; + else + *limit = smu->min_power_limit; break; default: return -EINVAL; @@ -2878,7 +2881,14 @@ static int smu_set_power_limit(void *handle, uint32_t limit) if (smu->ppt_funcs->set_power_limit) return smu->ppt_funcs->set_power_limit(smu, limit_type, limit); - if ((limit > smu->max_power_limit) || (limit < smu->min_power_limit)) { + if (amdgpu_ignore_min_pcap) { + if ((limit > smu->max_power_limit)) { + dev_err(smu->adev->dev, + "New power limit (%d) is over the max allowed %d\n", + limit, smu->max_power_limit); + return -EINVAL; + } + } else if ((limit > smu->max_power_limit) || (limit < smu->min_power_limit)) { dev_err(smu->adev->dev, "New power limit (%d) is out of range [%d,%d]\n", limit, smu->min_power_limit, smu->max_power_limit); diff --git a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h index 8846eb9ce2a4..c7d5d782e3f9 100644 --- a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h @@ -21,6 +21,9 @@ #define BMG_PACKAGE_POWER_SKU XE_REG(0x138098) #define BMG_PACKAGE_POWER_SKU_UNIT XE_REG(0x1380dc) #define BMG_PACKAGE_ENERGY_STATUS XE_REG(0x138120) +#define BMG_FAN_1_SPEED XE_REG(0x138140) +#define BMG_FAN_2_SPEED XE_REG(0x138170) +#define BMG_FAN_3_SPEED XE_REG(0x1381a0) #define BMG_VRAM_TEMPERATURE XE_REG(0x1382c0) #define BMG_PACKAGE_TEMPERATURE XE_REG(0x138434) #define BMG_PACKAGE_RAPL_LIMIT XE_REG(0x138440) diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 0482f26aa480..0564164935c6 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -314,6 +314,8 @@ struct xe_device { u8 has_atomic_enable_pte_bit:1; /** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */ u8 has_device_atomics_on_smem:1; + /** @info.has_fan_control: Device supports fan control */ + u8 has_fan_control:1; /** @info.has_flat_ccs: Whether flat CCS metadata is used */ u8 has_flat_ccs:1; /** @info.has_heci_cscfi: device has heci cscfi */ diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 48d80ffdf7bb..eb293aec36a0 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -27,6 +28,7 @@ enum xe_hwmon_reg { REG_PKG_POWER_SKU_UNIT, REG_GT_PERF_STATUS, REG_PKG_ENERGY_STATUS, + REG_FAN_SPEED, }; enum xe_hwmon_reg_operation { @@ -42,6 +44,13 @@ enum xe_hwmon_channel { CHANNEL_MAX, }; +enum xe_fan_channel { + FAN_1, + FAN_2, + FAN_3, + FAN_MAX, +}; + /* * SF_* - scale factors for particular quantities according to hwmon spec. */ @@ -61,6 +70,16 @@ struct xe_hwmon_energy_info { long accum_energy; }; +/** + * struct xe_hwmon_fan_info - to cache previous fan reading + */ +struct xe_hwmon_fan_info { + /** @reg_val_prev: previous fan reg val */ + u32 reg_val_prev; + /** @time_prev: previous timestamp */ + u64 time_prev; +}; + /** * struct xe_hwmon - xe hwmon data structure */ @@ -79,6 +98,8 @@ struct xe_hwmon { int scl_shift_time; /** @ei: Energy info for energyN_input */ struct xe_hwmon_energy_info ei[CHANNEL_MAX]; + /** @fi: Fan info for fanN_input */ + struct xe_hwmon_fan_info fi[FAN_MAX]; }; static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg, @@ -144,6 +165,14 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg return PCU_CR_PACKAGE_ENERGY_STATUS; } break; + case REG_FAN_SPEED: + if (channel == FAN_1) + return BMG_FAN_1_SPEED; + else if (channel == FAN_2) + return BMG_FAN_2_SPEED; + else if (channel == FAN_3) + return BMG_FAN_3_SPEED; + break; default: drm_warn(&xe->drm, "Unknown xe hwmon reg id: %d\n", hwmon_reg); break; @@ -454,6 +483,7 @@ static const struct hwmon_channel_info * const hwmon_info[] = { HWMON_CHANNEL_INFO(curr, HWMON_C_LABEL, HWMON_C_CRIT | HWMON_C_LABEL), HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_LABEL, HWMON_I_INPUT | HWMON_I_LABEL), HWMON_CHANNEL_INFO(energy, HWMON_E_INPUT | HWMON_E_LABEL, HWMON_E_INPUT | HWMON_E_LABEL), + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT, HWMON_F_INPUT, HWMON_F_INPUT), NULL }; @@ -480,6 +510,19 @@ static int xe_hwmon_pcode_write_i1(const struct xe_hwmon *hwmon, u32 uval) (uval & POWER_SETUP_I1_DATA_MASK)); } +static int xe_hwmon_pcode_read_fan_control(const struct xe_hwmon *hwmon, u32 subcmd, u32 *uval) +{ + struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + + /* Platforms that don't return correct value */ + if (hwmon->xe->info.platform == XE_DG2 && subcmd == FSC_READ_NUM_FANS) { + *uval = 2; + return 0; + } + + return xe_pcode_read(root_tile, PCODE_MBOX(FAN_SPEED_CONTROL, subcmd, 0), uval, NULL); +} + static int xe_hwmon_power_curr_crit_read(struct xe_hwmon *hwmon, int channel, long *value, u32 scale_factor) { @@ -705,6 +748,75 @@ xe_hwmon_energy_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) } } +static umode_t +xe_hwmon_fan_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) +{ + u32 uval; + + if (!hwmon->xe->info.has_fan_control) + return 0; + + switch (attr) { + case hwmon_fan_input: + if (xe_hwmon_pcode_read_fan_control(hwmon, FSC_READ_NUM_FANS, &uval)) + return 0; + + return channel < uval ? 0444 : 0; + default: + return 0; + } +} + +static int +xe_hwmon_fan_input_read(struct xe_hwmon *hwmon, int channel, long *val) +{ + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); + struct xe_hwmon_fan_info *fi = &hwmon->fi[channel]; + u64 rotations, time_now, time; + u32 reg_val; + int ret = 0; + + mutex_lock(&hwmon->hwmon_lock); + + reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_FAN_SPEED, channel)); + time_now = get_jiffies_64(); + + /* + * HW register value is accumulated count of pulses from PWM fan with the scale + * of 2 pulses per rotation. + */ + rotations = (reg_val - fi->reg_val_prev) / 2; + + time = jiffies_delta_to_msecs(time_now - fi->time_prev); + if (unlikely(!time)) { + ret = -EAGAIN; + goto unlock; + } + + /* + * Calculate fan speed in RPM by time averaging two subsequent readings in minutes. + * RPM = number of rotations * msecs per minute / time in msecs + */ + *val = DIV_ROUND_UP_ULL(rotations * (MSEC_PER_SEC * 60), time); + + fi->reg_val_prev = reg_val; + fi->time_prev = time_now; +unlock: + mutex_unlock(&hwmon->hwmon_lock); + return ret; +} + +static int +xe_hwmon_fan_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) +{ + switch (attr) { + case hwmon_fan_input: + return xe_hwmon_fan_input_read(hwmon, channel, val); + default: + return -EOPNOTSUPP; + } +} + static umode_t xe_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type, u32 attr, int channel) @@ -730,6 +842,9 @@ xe_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_types type, case hwmon_energy: ret = xe_hwmon_energy_is_visible(hwmon, attr, channel); break; + case hwmon_fan: + ret = xe_hwmon_fan_is_visible(hwmon, attr, channel); + break; default: ret = 0; break; @@ -765,6 +880,9 @@ xe_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, case hwmon_energy: ret = xe_hwmon_energy_read(hwmon, attr, channel, val); break; + case hwmon_fan: + ret = xe_hwmon_fan_read(hwmon, attr, channel, val); + break; default: ret = -EOPNOTSUPP; break; @@ -842,7 +960,7 @@ static void xe_hwmon_get_preregistration_info(struct xe_hwmon *hwmon) { struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); - long energy; + long energy, fan_speed; u64 val_sku_unit = 0; int channel; struct xe_reg pkg_power_sku_unit; @@ -866,6 +984,11 @@ xe_hwmon_get_preregistration_info(struct xe_hwmon *hwmon) for (channel = 0; channel < CHANNEL_MAX; channel++) if (xe_hwmon_is_visible(hwmon, hwmon_energy, hwmon_energy_input, channel)) xe_hwmon_energy_get(hwmon, channel, &energy); + + /* Initialize 'struct xe_hwmon_fan_info' with initial fan register reading. */ + for (channel = 0; channel < FAN_MAX; channel++) + if (xe_hwmon_is_visible(hwmon, hwmon_fan, hwmon_fan_input, channel)) + xe_hwmon_fan_input_read(hwmon, channel, &fan_speed); } static void xe_hwmon_mutex_destroy(void *arg) diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index f4d108dc49b1..977d0b07b9d7 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -62,6 +62,7 @@ struct xe_device_desc { u8 is_dgfx:1; u8 has_display:1; + u8 has_fan_control:1; u8 has_heci_gscfi:1; u8 has_heci_cscfi:1; u8 has_llc:1; @@ -303,6 +304,7 @@ static const struct xe_device_desc dg2_desc = { DG2_FEATURES, .has_display = true, + .has_fan_control = true, }; static const __maybe_unused struct xe_device_desc pvc_desc = { @@ -337,6 +339,7 @@ static const struct xe_device_desc bmg_desc = { PLATFORM(BATTLEMAGE), .dma_mask_size = 46, .has_display = true, + .has_fan_control = true, .has_heci_cscfi = 1, }; @@ -576,6 +579,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.dma_mask_size = desc->dma_mask_size; xe->info.is_dgfx = desc->is_dgfx; + xe->info.has_fan_control = desc->has_fan_control; xe->info.has_heci_gscfi = desc->has_heci_gscfi; xe->info.has_heci_cscfi = desc->has_heci_cscfi; xe->info.has_llc = desc->has_llc; diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h index 2bae9afdbd35..e622ae17f08d 100644 --- a/drivers/gpu/drm/xe/xe_pcode_api.h +++ b/drivers/gpu/drm/xe/xe_pcode_api.h @@ -49,6 +49,9 @@ /* Domain IDs (param2) */ #define PCODE_MBOX_DOMAIN_HBM 0x2 +#define FAN_SPEED_CONTROL 0x7D +#define FSC_READ_NUM_FANS 0x4 + #define PCODE_SCRATCH(x) XE_REG(0x138320 + ((x) * 4)) /* PCODE_SCRATCH0 */ #define AUXINFO_REG_OFFSET REG_GENMASK(17, 15) diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_client.c b/drivers/hid/amd-sfh-hid/amd_sfh_client.c index 3438d392920f..867019955b10 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_client.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_client.c @@ -146,6 +146,8 @@ static const char *get_sensor_name(int idx) return "gyroscope"; case mag_idx: return "magnetometer"; + case tms_idx: + return "tablet-mode-switch"; case als_idx: case ACS_IDX: /* ambient color sensor */ return "ALS"; diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index 1c1fd63330c9..7fd486a279b5 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -29,6 +29,7 @@ #define ACEL_EN BIT(0) #define GYRO_EN BIT(1) #define MAGNO_EN BIT(2) +#define TMS_EN BIT(15) #define HPD_EN BIT(16) #define ALS_EN BIT(19) #define ACS_EN BIT(22) @@ -232,6 +233,9 @@ int amd_mp2_get_sensor_num(struct amd_mp2_dev *privdata, u8 *sensor_id) if (MAGNO_EN & activestatus) sensor_id[num_of_sensors++] = mag_idx; + if (TMS_EN & activestatus) + sensor_id[num_of_sensors++] = tms_idx; + if (ALS_EN & activestatus) sensor_id[num_of_sensors++] = als_idx; diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h index 05e400a4a83e..617f0265b707 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.h @@ -79,6 +79,7 @@ enum sensor_idx { accel_idx = 0, gyro_idx = 1, mag_idx = 2, + tms_idx = 15, als_idx = 19 }; diff --git a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c index ef1f9be8b893..516a07bf3be6 100644 --- a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c +++ b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.c @@ -47,6 +47,11 @@ static int get_report_descriptor(int sensor_idx, u8 *rep_desc) memcpy(rep_desc, comp3_report_descriptor, sizeof(comp3_report_descriptor)); break; + case tms_idx: /* tablet mode switch */ + memset(rep_desc, 0, sizeof(tms_report_descriptor)); + memcpy(rep_desc, tms_report_descriptor, + sizeof(tms_report_descriptor)); + break; case als_idx: /* ambient light sensor */ case ACS_IDX: /* ambient color sensor */ memset(rep_desc, 0, sizeof(als_report_descriptor)); @@ -97,6 +102,16 @@ static u32 get_descr_sz(int sensor_idx, int descriptor_name) return sizeof(struct magno_feature_report); } break; + case tms_idx: + switch (descriptor_name) { + case descr_size: + return sizeof(tms_report_descriptor); + case input_size: + return sizeof(struct tms_input_report); + case feature_size: + return sizeof(struct tms_feature_report); + } + break; case als_idx: case ACS_IDX: /* ambient color sensor */ switch (descriptor_name) { @@ -140,6 +155,7 @@ static u8 get_feature_report(int sensor_idx, int report_id, u8 *feature_report) struct accel3_feature_report acc_feature; struct gyro_feature_report gyro_feature; struct magno_feature_report magno_feature; + struct tms_feature_report tms_feature; struct hpd_feature_report hpd_feature; struct als_feature_report als_feature; u8 report_size = 0; @@ -175,6 +191,11 @@ static u8 get_feature_report(int sensor_idx, int report_id, u8 *feature_report) memcpy(feature_report, &magno_feature, sizeof(magno_feature)); report_size = sizeof(magno_feature); break; + case tms_idx: /* tablet mode switch */ + get_common_features(&tms_feature.common_property, report_id); + memcpy(feature_report, &tms_feature, sizeof(tms_feature)); + report_size = sizeof(tms_feature); + break; case als_idx: /* ambient light sensor */ case ACS_IDX: /* ambient color sensor */ get_common_features(&als_feature.common_property, report_id); @@ -214,6 +235,7 @@ static u8 get_input_report(u8 current_index, int sensor_idx, int report_id, struct accel3_input_report acc_input; struct gyro_input_report gyro_input; struct hpd_input_report hpd_input; + struct tms_input_report tms_input; struct als_input_report als_input; struct hpd_status hpdstatus; u8 report_size = 0; @@ -247,6 +269,11 @@ static u8 get_input_report(u8 current_index, int sensor_idx, int report_id, memcpy(input_report, &magno_input, sizeof(magno_input)); report_size = sizeof(magno_input); break; +case tms_idx: /* tablet mode switch */ + get_common_inputs(&tms_input.common_property, report_id); + report_size = sizeof(tms_input); + memcpy(input_report, &tms_input, sizeof(tms_input)); + break; case als_idx: /* Als */ case ACS_IDX: /* ambient color sensor */ get_common_inputs(&als_input.common_property, report_id); diff --git a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.h b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.h index 882434b1501f..afcdac989cb6 100644 --- a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.h +++ b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_desc.h @@ -114,4 +114,12 @@ struct hpd_input_report { u8 human_presence; } __packed; +struct tms_feature_report { + struct common_feature_property common_property; +} __packed; + +struct tms_input_report { + struct common_input_property common_property; +} __packed; + #endif diff --git a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_report_desc.h b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_report_desc.h index 67ec2d6a417d..4dc87d684776 100644 --- a/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_report_desc.h +++ b/drivers/hid/amd-sfh-hid/hid_descriptor/amd_sfh_hid_report_desc.h @@ -665,6 +665,26 @@ static const u8 als_report_descriptor[] = { 0xC0 /* HID end collection */ }; +/* TABLET MODE SWITCH */ +__maybe_unused // Used by sfh1.0, but not yet implemented in sfh1.1 +static const u8 tms_report_descriptor[] = { +0x06, 0x43, 0xFF, // Usage Page (Vendor Defined 0xFF43) +0x0A, 0x02, 0x02, // Usage (0x0202) +0xA1, 0x01, // Collection (Application) +0x85, 0x11, // Report ID (17) +0x15, 0x00, // Logical Minimum (0) +0x25, 0x01, // Logical Maximum (1) +0x35, 0x00, // Physical Minimum (0) +0x45, 0x01, // Physical Maximum (1) +0x65, 0x00, // Unit (None) +0x55, 0x00, // Unit Exponent (0) +0x75, 0x01, // Report Size (1) +0x95, 0x98, // Report Count (-104) +0x81, 0x03, // Input (Const,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) +0x91, 0x03, // Output (Const,Var,Abs,No Wrap,Linear,Preferred State,No Null Position,Non-volatile) +0xC1, 0x00, // End Collection +}; + /* BIOMETRIC PRESENCE*/ static const u8 hpd_report_descriptor[] = { 0x05, 0x20, /* Usage page */ diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index b5cbb57ee5f6..a0f7fa1518c6 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -46,6 +46,7 @@ struct evdev_client { struct fasync_struct *fasync; struct evdev *evdev; struct list_head node; + struct rcu_head rcu; enum input_clock_type clk_type; bool revoked; unsigned long *evmasks[EV_CNT]; @@ -368,13 +369,22 @@ static void evdev_attach_client(struct evdev *evdev, spin_unlock(&evdev->client_lock); } +static void evdev_reclaim_client(struct rcu_head *rp) +{ + struct evdev_client *client = container_of(rp, struct evdev_client, rcu); + unsigned int i; + for (i = 0; i < EV_CNT; ++i) + bitmap_free(client->evmasks[i]); + kvfree(client); +} + static void evdev_detach_client(struct evdev *evdev, struct evdev_client *client) { spin_lock(&evdev->client_lock); list_del_rcu(&client->node); spin_unlock(&evdev->client_lock); - synchronize_rcu(); + call_rcu(&client->rcu, evdev_reclaim_client); } static int evdev_open_device(struct evdev *evdev) @@ -427,7 +437,6 @@ static int evdev_release(struct inode *inode, struct file *file) { struct evdev_client *client = file->private_data; struct evdev *evdev = client->evdev; - unsigned int i; mutex_lock(&evdev->mutex); @@ -439,11 +448,6 @@ static int evdev_release(struct inode *inode, struct file *file) evdev_detach_client(evdev, client); - for (i = 0; i < EV_CNT; ++i) - bitmap_free(client->evmasks[i]); - - kvfree(client); - evdev_close_device(evdev); return 0; @@ -486,7 +490,6 @@ static int evdev_open(struct inode *inode, struct file *file) err_free_client: evdev_detach_client(evdev, client); - kvfree(client); return error; } diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 9dfdb63220d7..91aeee72a15e 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -3284,6 +3284,11 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad; } +#ifdef CONFIG_CACHY + set_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); + set_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); +#endif + ret = crypt_ctr_cipher(ti, argv[0], argv[1]); if (ret < 0) goto bad; diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig index 331b8e535e5b..80dabeebf580 100644 --- a/drivers/media/v4l2-core/Kconfig +++ b/drivers/media/v4l2-core/Kconfig @@ -40,6 +40,11 @@ config VIDEO_TUNER config V4L2_JPEG_HELPER tristate +config V4L2_LOOPBACK + tristate "V4L2 loopback device" + help + V4L2 loopback device + # Used by drivers that need v4l2-h264.ko config V4L2_H264 tristate diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile index 2177b9d63a8f..c179507cedc4 100644 --- a/drivers/media/v4l2-core/Makefile +++ b/drivers/media/v4l2-core/Makefile @@ -33,5 +33,7 @@ obj-$(CONFIG_V4L2_JPEG_HELPER) += v4l2-jpeg.o obj-$(CONFIG_V4L2_MEM2MEM_DEV) += v4l2-mem2mem.o obj-$(CONFIG_V4L2_VP9) += v4l2-vp9.o +obj-$(CONFIG_V4L2_LOOPBACK) += v4l2loopback.o + obj-$(CONFIG_VIDEO_TUNER) += tuner.o obj-$(CONFIG_VIDEO_DEV) += v4l2-dv-timings.o videodev.o diff --git a/drivers/media/v4l2-core/v4l2loopback.c b/drivers/media/v4l2-core/v4l2loopback.c new file mode 100644 index 000000000000..5c8b70981afa --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback.c @@ -0,0 +1,3292 @@ +/* -*- c-file-style: "linux" -*- */ +/* + * v4l2loopback.c -- video4linux2 loopback driver + * + * Copyright (C) 2005-2009 Vasily Levin (vasaka@gmail.com) + * Copyright (C) 2010-2023 IOhannes m zmoelnig (zmoelnig@iem.at) + * Copyright (C) 2011 Stefan Diewald (stefan.diewald@mytum.de) + * Copyright (C) 2012 Anton Novikov (random.plant@gmail.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "v4l2loopback.h" + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) +#error This module is not supported on kernels before 4.0.0. +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#define strscpy strlcpy +#endif + +#if defined(timer_setup) && defined(from_timer) +#define HAVE_TIMER_SETUP +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0) +#define VFL_TYPE_VIDEO VFL_TYPE_GRABBER +#endif + +#define V4L2LOOPBACK_VERSION_CODE \ + KERNEL_VERSION(V4L2LOOPBACK_VERSION_MAJOR, V4L2LOOPBACK_VERSION_MINOR, \ + V4L2LOOPBACK_VERSION_BUGFIX) + +MODULE_DESCRIPTION("V4L2 loopback video device"); +MODULE_AUTHOR("Vasily Levin, " + "IOhannes m zmoelnig ," + "Stefan Diewald," + "Anton Novikov" + "et al."); +#ifdef SNAPSHOT_VERSION +MODULE_VERSION(__stringify(SNAPSHOT_VERSION)); +#else +MODULE_VERSION("" __stringify(V4L2LOOPBACK_VERSION_MAJOR) "." __stringify( + V4L2LOOPBACK_VERSION_MINOR) "." __stringify(V4L2LOOPBACK_VERSION_BUGFIX)); +#endif +MODULE_LICENSE("GPL"); + +/* + * helpers + */ +#define dprintk(fmt, args...) \ + do { \ + if (debug > 0) { \ + printk(KERN_INFO "v4l2-loopback[" __stringify( \ + __LINE__) "], pid(%d): " fmt, \ + task_pid_nr(current), ##args); \ + } \ + } while (0) + +#define MARK() \ + do { \ + if (debug > 1) { \ + printk(KERN_INFO "%s:%d[%s], pid(%d)\n", __FILE__, \ + __LINE__, __func__, task_pid_nr(current)); \ + } \ + } while (0) + +#define dprintkrw(fmt, args...) \ + do { \ + if (debug > 2) { \ + printk(KERN_INFO "v4l2-loopback[" __stringify( \ + __LINE__) "], pid(%d): " fmt, \ + task_pid_nr(current), ##args); \ + } \ + } while (0) + +static inline void v4l2l_get_timestamp(struct v4l2_buffer *b) +{ + struct timespec64 ts; + ktime_get_ts64(&ts); + + b->timestamp.tv_sec = ts.tv_sec; + b->timestamp.tv_usec = (ts.tv_nsec / NSEC_PER_USEC); + b->flags |= V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + b->flags &= ~V4L2_BUF_FLAG_TIMESTAMP_COPY; +} + +#if BITS_PER_LONG == 32 +#include /* do_div() for 64bit division */ +static inline int v4l2l_mod64(const s64 A, const u32 B) +{ + u64 a = (u64)A; + u32 b = B; + + if (A > 0) + return do_div(a, b); + a = -A; + return -do_div(a, b); +} +#else +static inline int v4l2l_mod64(const s64 A, const u32 B) +{ + return A % B; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) +typedef unsigned __poll_t; +#endif + +/* module constants + * can be overridden during he build process using something like + * make KCPPFLAGS="-DMAX_DEVICES=100" + */ + +/* maximum number of v4l2loopback devices that can be created */ +#ifndef MAX_DEVICES +#define MAX_DEVICES 8 +#endif + +/* whether the default is to announce capabilities exclusively or not */ +#ifndef V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +#define V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS 0 +#endif + +/* when a producer is considered to have gone stale */ +#ifndef MAX_TIMEOUT +#define MAX_TIMEOUT (100 * 1000) /* in msecs */ +#endif + +/* max buffers that can be mapped, actually they + * are all mapped to max_buffers buffers */ +#ifndef MAX_BUFFERS +#define MAX_BUFFERS 32 +#endif + +/* module parameters */ +static int debug = 0; +module_param(debug, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(debug, "debugging level (higher values == more verbose)"); + +#define V4L2LOOPBACK_DEFAULT_MAX_BUFFERS 2 +static int max_buffers = V4L2LOOPBACK_DEFAULT_MAX_BUFFERS; +module_param(max_buffers, int, S_IRUGO); +MODULE_PARM_DESC(max_buffers, + "how many buffers should be allocated [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_MAX_BUFFERS) "]"); + +/* how many times a device can be opened + * the per-module default value can be overridden on a per-device basis using + * the /sys/devices interface + * + * note that max_openers should be at least 2 in order to get a working system: + * one opener for the producer and one opener for the consumer + * however, we leave that to the user + */ +#define V4L2LOOPBACK_DEFAULT_MAX_OPENERS 10 +static int max_openers = V4L2LOOPBACK_DEFAULT_MAX_OPENERS; +module_param(max_openers, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC( + max_openers, + "how many users can open the loopback device [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_MAX_OPENERS) "]"); + +static int devices = -1; +module_param(devices, int, 0); +MODULE_PARM_DESC(devices, "how many devices should be created"); + +static int video_nr[MAX_DEVICES] = { [0 ...(MAX_DEVICES - 1)] = -1 }; +module_param_array(video_nr, int, NULL, 0444); +MODULE_PARM_DESC(video_nr, + "video device numbers (-1=auto, 0=/dev/video0, etc.)"); + +static char *card_label[MAX_DEVICES]; +module_param_array(card_label, charp, NULL, 0000); +MODULE_PARM_DESC(card_label, "card labels for each device"); + +static bool exclusive_caps[MAX_DEVICES] = { + [0 ...(MAX_DEVICES - 1)] = V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS +}; +module_param_array(exclusive_caps, bool, NULL, 0444); +/* FIXXME: wording */ +MODULE_PARM_DESC( + exclusive_caps, + "whether to announce OUTPUT/CAPTURE capabilities exclusively or not [DEFAULT: " __stringify( + V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS) "]"); + +/* format specifications */ +#define V4L2LOOPBACK_SIZE_MIN_WIDTH 2 +#define V4L2LOOPBACK_SIZE_MIN_HEIGHT 1 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH 8192 +#define V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT 8192 + +#define V4L2LOOPBACK_SIZE_DEFAULT_WIDTH 640 +#define V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT 480 + +static int max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; +module_param(max_width, int, S_IRUGO); +MODULE_PARM_DESC(max_width, + "maximum allowed frame width [DEFAULT: " __stringify( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH) "]"); +static int max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; +module_param(max_height, int, S_IRUGO); +MODULE_PARM_DESC(max_height, + "maximum allowed frame height [DEFAULT: " __stringify( + V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT) "]"); + +static DEFINE_IDR(v4l2loopback_index_idr); +static DEFINE_MUTEX(v4l2loopback_ctl_mutex); + +/* frame intervals */ +#define V4L2LOOPBACK_FRAME_INTERVAL_MAX __UINT32_MAX__ +#define V4L2LOOPBACK_FPS_DEFAULT 30 +#define V4L2LOOPBACK_FPS_MAX 1000 + +/* control IDs */ +#define V4L2LOOPBACK_CID_BASE (V4L2_CID_USER_BASE | 0xf000) +#define CID_KEEP_FORMAT (V4L2LOOPBACK_CID_BASE + 0) +#define CID_SUSTAIN_FRAMERATE (V4L2LOOPBACK_CID_BASE + 1) +#define CID_TIMEOUT (V4L2LOOPBACK_CID_BASE + 2) +#define CID_TIMEOUT_IMAGE_IO (V4L2LOOPBACK_CID_BASE + 3) + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl); +static const struct v4l2_ctrl_ops v4l2loopback_ctrl_ops = { + .s_ctrl = v4l2loopback_s_ctrl, +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_keepformat = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_KEEP_FORMAT, + .name = "keep_format", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_sustainframerate = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_SUSTAIN_FRAMERATE, + .name = "sustain_framerate", + .type = V4L2_CTRL_TYPE_BOOLEAN, + .min = 0, + .max = 1, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeout = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT, + .name = "timeout", + .type = V4L2_CTRL_TYPE_INTEGER, + .min = 0, + .max = MAX_TIMEOUT, + .step = 1, + .def = 0, + // clang-format on +}; +static const struct v4l2_ctrl_config v4l2loopback_ctrl_timeoutimageio = { + // clang-format off + .ops = &v4l2loopback_ctrl_ops, + .id = CID_TIMEOUT_IMAGE_IO, + .name = "timeout_image_io", + .type = V4L2_CTRL_TYPE_BUTTON, + .min = 0, + .max = 0, + .step = 0, + .def = 0, + // clang-format on +}; + +/* module structures */ +struct v4l2loopback_private { + int device_nr; +}; + +/* TODO(vasaka) use typenames which are common to kernel, but first find out if + * it is needed */ +/* struct keeping state and settings of loopback device */ + +struct v4l2l_buffer { + struct v4l2_buffer buffer; + struct list_head list_head; + atomic_t use_count; +}; + +struct v4l2_loopback_device { + struct v4l2_device v4l2_dev; + struct v4l2_ctrl_handler ctrl_handler; + struct video_device *vdev; + + /* loopback device-specific parameters */ + char card_label[32]; + bool announce_all_caps; /* announce both OUTPUT and CAPTURE capabilities + * when true; else announce OUTPUT when no + * writer is streaming, otherwise CAPTURE. */ + int max_openers; /* how many times can this device be opened */ + int min_width, max_width; + int min_height, max_height; + + /* pixel and stream format */ + struct v4l2_pix_format pix_format; + bool pix_format_has_valid_sizeimage; + struct v4l2_captureparm capture_param; + unsigned long frame_jiffies; + + /* ctrls */ + int keep_format; /* CID_KEEP_FORMAT; lock the format, do not free + * on close(), and when `!announce_all_caps` do NOT + * fall back to OUTPUT when no writers attached (clear + * `keep_format` to attach a new writer) */ + int sustain_framerate; /* CID_SUSTAIN_FRAMERATE; duplicate frames to maintain + (close to) nominal framerate */ + unsigned long timeout_jiffies; /* CID_TIMEOUT; 0 means disabled */ + int timeout_image_io; /* CID_TIMEOUT_IMAGE_IO; next opener will + * queue/dequeue the timeout image buffer */ + + /* buffers for OUTPUT and CAPTURE */ + u8 *image; /* pointer to actual buffers data */ + unsigned long image_size; /* number of bytes alloc'd for all buffers */ + struct v4l2l_buffer buffers[MAX_BUFFERS]; /* inner driver buffers */ + u32 buffer_count; /* should not be big, 4 is a good choice */ + u32 buffer_size; /* number of bytes alloc'd per buffer */ + u32 used_buffer_count; /* number of buffers allocated to openers */ + struct list_head outbufs_list; /* FIFO queue for OUTPUT buffers */ + u32 bufpos2index[MAX_BUFFERS]; /* mapping of `(position % used_buffers)` + * to `buffers[index]` */ + s64 write_position; /* sequence number of last 'displayed' buffer plus + * one */ + + /* synchronization between openers */ + atomic_t open_count; + struct mutex image_mutex; /* mutex for allocating image(s) and + * exchanging format tokens */ + spinlock_t lock; /* lock for the timeout and framerate timers */ + spinlock_t list_lock; /* lock for the OUTPUT buffer queue */ + wait_queue_head_t read_event; + u32 format_tokens; /* tokens to 'set format' for OUTPUT, CAPTURE, or + * timeout buffers */ + u32 stream_tokens; /* tokens to 'start' OUTPUT, CAPTURE, or timeout + * stream */ + + /* sustain framerate */ + struct timer_list sustain_timer; + unsigned int reread_count; + + /* timeout */ + u8 *timeout_image; /* copied to outgoing buffers when timeout passes */ + struct v4l2l_buffer timeout_buffer; + u32 timeout_buffer_size; /* number bytes alloc'd for timeout buffer */ + struct timer_list timeout_timer; + int timeout_happened; +}; + +enum v4l2l_io_method { + V4L2L_IO_NONE = 0, + V4L2L_IO_MMAP = 1, + V4L2L_IO_FILE = 2, + V4L2L_IO_TIMEOUT = 3, +}; + +/* struct keeping state and type of opener */ +struct v4l2_loopback_opener { + u32 format_token; /* token (if any) for type used in call to S_FMT or + * REQBUFS */ + u32 stream_token; /* token (if any) for type used in call to STREAMON */ + u32 buffer_count; /* number of buffers (if any) that opener acquired via + * REQBUFS */ + s64 read_position; /* sequence number of the next 'captured' frame */ + unsigned int reread_count; + enum v4l2l_io_method io_method; + + struct v4l2_fh fh; +}; + +#define fh_to_opener(ptr) container_of((ptr), struct v4l2_loopback_opener, fh) + +/* this is heavily inspired by the bttv driver found in the linux kernel */ +struct v4l2l_format { + char *name; + int fourcc; /* video4linux 2 */ + int depth; /* bit/pixel */ + int flags; +}; +/* set the v4l2l_format.flags to PLANAR for non-packed formats */ +#define FORMAT_FLAGS_PLANAR 0x01 +#define FORMAT_FLAGS_COMPRESSED 0x02 + +#include "v4l2loopback_formats.h" + +#ifndef V4L2_TYPE_IS_CAPTURE +#define V4L2_TYPE_IS_CAPTURE(type) \ + ((type) == V4L2_BUF_TYPE_VIDEO_CAPTURE || \ + (type) == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) +#endif /* V4L2_TYPE_IS_CAPTURE */ +#ifndef V4L2_TYPE_IS_OUTPUT +#define V4L2_TYPE_IS_OUTPUT(type) \ + ((type) == V4L2_BUF_TYPE_VIDEO_OUTPUT || \ + (type) == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) +#endif /* V4L2_TYPE_IS_OUTPUT */ + +/* token values for privilege to set format or start/stop stream */ +#define V4L2L_TOKEN_CAPTURE 0x01 +#define V4L2L_TOKEN_OUTPUT 0x02 +#define V4L2L_TOKEN_TIMEOUT 0x04 +#define V4L2L_TOKEN_MASK \ + (V4L2L_TOKEN_CAPTURE | V4L2L_TOKEN_OUTPUT | V4L2L_TOKEN_TIMEOUT) + +/* helpers for token exchange and token status */ +#define token_from_type(type) \ + (V4L2_TYPE_IS_CAPTURE(type) ? V4L2L_TOKEN_CAPTURE : V4L2L_TOKEN_OUTPUT) +#define acquire_token(dev, opener, label, token) \ + do { \ + (opener)->label##_token = token; \ + (dev)->label##_tokens &= ~token; \ + } while (0) +#define release_token(dev, opener, label) \ + do { \ + (dev)->label##_tokens |= (opener)->label##_token; \ + (opener)->label##_token = 0; \ + } while (0) +#define has_output_token(token) (token & V4L2L_TOKEN_OUTPUT) +#define has_capture_token(token) (token & V4L2L_TOKEN_CAPTURE) +#define has_no_owners(dev) ((~((dev)->format_tokens) & V4L2L_TOKEN_MASK) == 0) +#define has_other_owners(opener, dev) \ + (~((dev)->format_tokens ^ (opener)->format_token) & V4L2L_TOKEN_MASK) +#define need_timeout_buffer(dev, token) \ + ((dev)->timeout_jiffies > 0 || (token) & V4L2L_TOKEN_TIMEOUT) + +static const unsigned int FORMATS = ARRAY_SIZE(formats); + +static char *fourcc2str(unsigned int fourcc, char buf[5]) +{ + buf[0] = (fourcc >> 0) & 0xFF; + buf[1] = (fourcc >> 8) & 0xFF; + buf[2] = (fourcc >> 16) & 0xFF; + buf[3] = (fourcc >> 24) & 0xFF; + buf[4] = 0; + + return buf; +} + +static const struct v4l2l_format *format_by_fourcc(int fourcc) +{ + unsigned int i; + char buf[5]; + + for (i = 0; i < FORMATS; i++) { + if (formats[i].fourcc == fourcc) + return formats + i; + } + + dprintk("unsupported format '%4s'\n", fourcc2str(fourcc, buf)); + return NULL; +} + +static void pix_format_set_size(struct v4l2_pix_format *f, + const struct v4l2l_format *fmt, + unsigned int width, unsigned int height) +{ + f->width = width; + f->height = height; + + if (fmt->flags & FORMAT_FLAGS_PLANAR) { + f->bytesperline = width; /* Y plane */ + f->sizeimage = (width * height * fmt->depth) >> 3; + } else if (fmt->flags & FORMAT_FLAGS_COMPRESSED) { + /* doesn't make sense for compressed formats */ + f->bytesperline = 0; + f->sizeimage = (width * height * fmt->depth) >> 3; + } else { + f->bytesperline = (width * fmt->depth) >> 3; + f->sizeimage = height * f->bytesperline; + } +} + +static int v4l2l_fill_format(struct v4l2_format *fmt, const u32 minwidth, + const u32 maxwidth, const u32 minheight, + const u32 maxheight) +{ + u32 width = fmt->fmt.pix.width, height = fmt->fmt.pix.height; + u32 pixelformat = fmt->fmt.pix.pixelformat; + struct v4l2_format fmt0 = *fmt; + u32 bytesperline = 0, sizeimage = 0; + + if (!width) + width = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + if (!height) + height = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + width = clamp_val(width, minwidth, maxwidth); + height = clamp_val(height, minheight, maxheight); + + /* sets: width,height,pixelformat,bytesperline,sizeimage */ + if (!(V4L2_TYPE_IS_MULTIPLANAR(fmt0.type))) { + fmt0.fmt.pix.bytesperline = 0; + fmt0.fmt.pix.sizeimage = 0; + } + + if (0) { + ; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0) + } else if (!v4l2_fill_pixfmt(&fmt0.fmt.pix, pixelformat, width, + height)) { + ; + } else if (!v4l2_fill_pixfmt_mp(&fmt0.fmt.pix_mp, pixelformat, width, + height)) { + ; +#endif + } else { + const struct v4l2l_format *format = + format_by_fourcc(pixelformat); + if (!format) + return -EINVAL; + pix_format_set_size(&fmt0.fmt.pix, format, width, height); + fmt0.fmt.pix.pixelformat = format->fourcc; + } + + if (V4L2_TYPE_IS_MULTIPLANAR(fmt0.type)) { + *fmt = fmt0; + + if ((fmt->fmt.pix_mp.colorspace == V4L2_COLORSPACE_DEFAULT) || + (fmt->fmt.pix_mp.colorspace > V4L2_COLORSPACE_DCI_P3)) + fmt->fmt.pix_mp.colorspace = V4L2_COLORSPACE_SRGB; + if (V4L2_FIELD_ANY == fmt->fmt.pix_mp.field) + fmt->fmt.pix_mp.field = V4L2_FIELD_NONE; + } else { + bytesperline = fmt->fmt.pix.bytesperline; + sizeimage = fmt->fmt.pix.sizeimage; + + *fmt = fmt0; + + if (!fmt->fmt.pix.bytesperline) + fmt->fmt.pix.bytesperline = bytesperline; + if (!fmt->fmt.pix.sizeimage) + fmt->fmt.pix.sizeimage = sizeimage; + + if ((fmt->fmt.pix.colorspace == V4L2_COLORSPACE_DEFAULT) || + (fmt->fmt.pix.colorspace > V4L2_COLORSPACE_DCI_P3)) + fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + if (V4L2_FIELD_ANY == fmt->fmt.pix.field) + fmt->fmt.pix.field = V4L2_FIELD_NONE; + } + + return 0; +} + +/* Checks if v4l2l_fill_format() has set a valid, fixed sizeimage val. */ +static bool v4l2l_pix_format_has_valid_sizeimage(struct v4l2_format *fmt) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0) + const struct v4l2_format_info *info; + + info = v4l2_format_info(fmt->fmt.pix.pixelformat); + if (info && info->mem_planes == 1) + return true; +#endif + + return false; +} + +static int pix_format_eq(const struct v4l2_pix_format *ref, + const struct v4l2_pix_format *tgt, int strict) +{ + /* check if the two formats are equivalent. + * ANY fields are handled gracefully + */ +#define _pix_format_eq0(x) \ + if (ref->x != tgt->x) \ + result = 0 +#define _pix_format_eq1(x, def) \ + do { \ + if ((def != tgt->x) && (ref->x != tgt->x)) { \ + printk(KERN_INFO #x " failed"); \ + result = 0; \ + } \ + } while (0) + int result = 1; + _pix_format_eq0(width); + _pix_format_eq0(height); + _pix_format_eq0(pixelformat); + if (!strict) + return result; + _pix_format_eq1(field, V4L2_FIELD_ANY); + _pix_format_eq0(bytesperline); + _pix_format_eq0(sizeimage); + _pix_format_eq1(colorspace, V4L2_COLORSPACE_DEFAULT); + return result; +} + +static void set_timeperframe(struct v4l2_loopback_device *dev, + struct v4l2_fract *tpf) +{ + if (!tpf->denominator && !tpf->numerator) { + tpf->numerator = 1; + tpf->denominator = V4L2LOOPBACK_FPS_DEFAULT; + } else if (tpf->numerator > + V4L2LOOPBACK_FRAME_INTERVAL_MAX * tpf->denominator) { + /* divide-by-zero or greater than maximum interval => min FPS */ + tpf->numerator = V4L2LOOPBACK_FRAME_INTERVAL_MAX; + tpf->denominator = 1; + } else if (tpf->numerator * V4L2LOOPBACK_FPS_MAX < tpf->denominator) { + /* zero or lower than minimum interval => max FPS */ + tpf->numerator = 1; + tpf->denominator = V4L2LOOPBACK_FPS_MAX; + } + + dev->capture_param.timeperframe = *tpf; + dev->frame_jiffies = + max(1UL, (msecs_to_jiffies(1000) * tpf->numerator) / + tpf->denominator); +} + +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd); + +/* device attributes */ +/* available via sysfs: /sys/devices/virtual/video4linux/video* */ + +static ssize_t attr_show_format(struct device *cd, + struct device_attribute *attr, char *buf) +{ + /* gets the current format as "FOURCC:WxH@f/s", e.g. "YUYV:320x240@1000/30" */ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + const struct v4l2_fract *tpf; + char buf4cc[5], buf_fps[32]; + + if (!dev || (has_no_owners(dev) && !dev->keep_format)) + return 0; + tpf = &dev->capture_param.timeperframe; + + fourcc2str(dev->pix_format.pixelformat, buf4cc); + if (tpf->numerator == 1) + snprintf(buf_fps, sizeof(buf_fps), "%u", tpf->denominator); + else + snprintf(buf_fps, sizeof(buf_fps), "%u/%u", tpf->denominator, + tpf->numerator); + return sprintf(buf, "%4s:%ux%u@%s\n", buf4cc, dev->pix_format.width, + dev->pix_format.height, buf_fps); +} + +static ssize_t attr_store_format(struct device *cd, + struct device_attribute *attr, const char *buf, + size_t len) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + int fps_num = 0, fps_den = 1; + + if (!dev) + return -ENODEV; + + /* only fps changing is supported */ + if (sscanf(buf, "@%u/%u", &fps_num, &fps_den) > 0) { + struct v4l2_fract f = { .numerator = fps_den, + .denominator = fps_num }; + set_timeperframe(dev, &f); + return len; + } + return -EINVAL; +} + +static DEVICE_ATTR(format, S_IRUGO | S_IWUSR, attr_show_format, + attr_store_format); + +static ssize_t attr_show_buffers(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + return sprintf(buf, "%u\n", dev->used_buffer_count); +} + +static DEVICE_ATTR(buffers, S_IRUGO, attr_show_buffers, NULL); + +static ssize_t attr_show_maxopeners(struct device *cd, + struct device_attribute *attr, char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + return sprintf(buf, "%d\n", dev->max_openers); +} + +static ssize_t attr_store_maxopeners(struct device *cd, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct v4l2_loopback_device *dev = NULL; + unsigned long curr = 0; + + if (kstrtoul(buf, 0, &curr)) + return -EINVAL; + + dev = v4l2loopback_cd2dev(cd); + if (!dev) + return -ENODEV; + + if (dev->max_openers == curr) + return len; + + if (curr > __INT_MAX__ || dev->open_count.counter > curr) { + /* request to limit to less openers as are currently attached to us */ + return -EINVAL; + } + + dev->max_openers = (int)curr; + + return len; +} + +static DEVICE_ATTR(max_openers, S_IRUGO | S_IWUSR, attr_show_maxopeners, + attr_store_maxopeners); + +static ssize_t attr_show_state(struct device *cd, struct device_attribute *attr, + char *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_cd2dev(cd); + + if (!dev) + return -ENODEV; + + if (!has_output_token(dev->stream_tokens) || dev->keep_format) { + return sprintf(buf, "capture\n"); + } else + return sprintf(buf, "output\n"); + + return -EAGAIN; +} + +static DEVICE_ATTR(state, S_IRUGO, attr_show_state, NULL); + +static void v4l2loopback_remove_sysfs(struct video_device *vdev) +{ +#define V4L2_SYSFS_DESTROY(x) device_remove_file(&vdev->dev, &dev_attr_##x) + + if (vdev) { + V4L2_SYSFS_DESTROY(format); + V4L2_SYSFS_DESTROY(buffers); + V4L2_SYSFS_DESTROY(max_openers); + V4L2_SYSFS_DESTROY(state); + /* ... */ + } +} + +static void v4l2loopback_create_sysfs(struct video_device *vdev) +{ + int res = 0; + +#define V4L2_SYSFS_CREATE(x) \ + res = device_create_file(&vdev->dev, &dev_attr_##x); \ + if (res < 0) \ + break + if (!vdev) + return; + do { + V4L2_SYSFS_CREATE(format); + V4L2_SYSFS_CREATE(buffers); + V4L2_SYSFS_CREATE(max_openers); + V4L2_SYSFS_CREATE(state); + /* ... */ + } while (0); + + if (res >= 0) + return; + dev_err(&vdev->dev, "%s error: %d\n", __func__, res); +} + +/* Event APIs */ + +#define V4L2LOOPBACK_EVENT_BASE (V4L2_EVENT_PRIVATE_START) +#define V4L2LOOPBACK_EVENT_OFFSET 0x08E00000 +#define V4L2_EVENT_PRI_CLIENT_USAGE \ + (V4L2LOOPBACK_EVENT_BASE + V4L2LOOPBACK_EVENT_OFFSET + 1) + +struct v4l2_event_client_usage { + __u32 count; +}; + +/* global module data */ +/* find a device based on it's device-number (e.g. '3' for /dev/video3) */ +struct v4l2loopback_lookup_cb_data { + int device_nr; + struct v4l2_loopback_device *device; +}; +static int v4l2loopback_lookup_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *device = ptr; + struct v4l2loopback_lookup_cb_data *cbdata = data; + if (cbdata && device && device->vdev) { + if (device->vdev->num == cbdata->device_nr) { + cbdata->device = device; + cbdata->device_nr = id; + return 1; + } + } + return 0; +} +static int v4l2loopback_lookup(int device_nr, + struct v4l2_loopback_device **device) +{ + struct v4l2loopback_lookup_cb_data data = { + .device_nr = device_nr, + .device = NULL, + }; + int err = idr_for_each(&v4l2loopback_index_idr, &v4l2loopback_lookup_cb, + &data); + if (1 == err) { + if (device) + *device = data.device; + return data.device_nr; + } + return -ENODEV; +} +#define v4l2loopback_get_vdev_nr(vdev) \ + ((struct v4l2loopback_private *)video_get_drvdata(vdev))->device_nr +static struct v4l2_loopback_device *v4l2loopback_cd2dev(struct device *cd) +{ + struct video_device *loopdev = to_video_device(cd); + int device_nr = v4l2loopback_get_vdev_nr(loopdev); + + return idr_find(&v4l2loopback_index_idr, device_nr); +} + +static struct v4l2_loopback_device *v4l2loopback_getdevice(struct file *f) +{ + struct v4l2loopback_private *ptr = video_drvdata(f); + int nr = ptr->device_nr; + + return idr_find(&v4l2loopback_index_idr, nr); +} + +/* forward declarations */ +static void client_usage_queue_event(struct video_device *vdev); +static bool any_buffers_mapped(struct v4l2_loopback_device *dev); +static int allocate_buffers(struct v4l2_loopback_device *dev, + struct v4l2_pix_format *pix_format); +static void init_buffers(struct v4l2_loopback_device *dev, u32 bytes_used, + u32 buffer_size); +static void free_buffers(struct v4l2_loopback_device *dev); +static int allocate_timeout_buffer(struct v4l2_loopback_device *dev); +static void free_timeout_buffer(struct v4l2_loopback_device *dev); +static void check_timers(struct v4l2_loopback_device *dev); +static const struct v4l2_file_operations v4l2_loopback_fops; +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops; + +/* V4L2 ioctl caps and params calls */ +/* returns device capabilities + * called on VIDIOC_QUERYCAP + */ +static int vidioc_querycap(struct file *file, void *fh, + struct v4l2_capability *cap) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + int device_nr = v4l2loopback_get_vdev_nr(dev->vdev); + __u32 capabilities = V4L2_CAP_STREAMING | V4L2_CAP_READWRITE; + + strscpy(cap->driver, "v4l2 loopback", sizeof(cap->driver)); + snprintf(cap->card, sizeof(cap->card), "%s", dev->card_label); + snprintf(cap->bus_info, sizeof(cap->bus_info), + "platform:v4l2loopback-%03d", device_nr); + + if (dev->announce_all_caps) { + capabilities |= V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_OUTPUT; + } else { + if (opener->io_method == V4L2L_IO_TIMEOUT || + (has_output_token(dev->stream_tokens) && + !dev->keep_format)) { + capabilities |= V4L2_CAP_VIDEO_OUTPUT; + } else + capabilities |= V4L2_CAP_VIDEO_CAPTURE; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + dev->vdev->device_caps = +#endif /* >=linux-4.7.0 */ + cap->device_caps = cap->capabilities = capabilities; + + cap->capabilities |= V4L2_CAP_DEVICE_CAPS; + + memset(cap->reserved, 0, sizeof(cap->reserved)); + return 0; +} + +static int vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + if (dev->keep_format || has_other_owners(opener, dev)) { + /* only current frame size supported */ + if (argp->pixel_format != dev->pix_format.pixelformat) + return -EINVAL; + + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->pix_format.width; + argp->discrete.height = dev->pix_format.height; + } else { + /* return continuous sizes if pixel format is supported */ + if (NULL == format_by_fourcc(argp->pixel_format)) + return -EINVAL; + + if (dev->min_width == dev->max_width && + dev->min_height == dev->max_height) { + argp->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + argp->discrete.width = dev->min_width; + argp->discrete.height = dev->min_height; + } else { + argp->type = V4L2_FRMSIZE_TYPE_CONTINUOUS; + + argp->stepwise.min_width = dev->min_width; + argp->stepwise.min_height = dev->min_height; + + argp->stepwise.max_width = dev->max_width; + argp->stepwise.max_height = dev->max_height; + + argp->stepwise.step_width = 1; + argp->stepwise.step_height = 1; + } + } + return 0; +} + +/* Test if the device is currently 'capable' of the buffer (stream) type when + * the `exclusive_caps` parameter is set. `keep_format` should lock the format + * and prevent free of buffers */ +static int check_buffer_capability(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener, + enum v4l2_buf_type type) +{ + /* short-circuit for (non-compliant) timeout image mode */ + if (opener->io_method == V4L2L_IO_TIMEOUT) + return 0; + if (dev->announce_all_caps) + return (type == V4L2_BUF_TYPE_VIDEO_CAPTURE || + type == V4L2_BUF_TYPE_VIDEO_OUTPUT) ? + 0 : + -EINVAL; + /* CAPTURE if opener has a capture format or a writer is streaming; + * else OUTPUT. */ + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (!(has_capture_token(opener->format_token) || + !has_output_token(dev->stream_tokens))) + return -EINVAL; + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (!(has_output_token(opener->format_token) || + has_output_token(dev->stream_tokens))) + return -EINVAL; + break; + default: + return -EINVAL; + } + return 0; +} +/* returns frameinterval (fps) for the set resolution + * called on VIDIOC_ENUM_FRAMEINTERVALS + */ +static int vidioc_enum_frameintervals(struct file *file, void *fh, + struct v4l2_frmivalenum *argp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + /* there can be only one... */ + if (argp->index) + return -EINVAL; + + if (dev->keep_format || has_other_owners(opener, dev)) { + /* keep_format also locks the frame rate */ + if (argp->width != dev->pix_format.width || + argp->height != dev->pix_format.height || + argp->pixel_format != dev->pix_format.pixelformat) + return -EINVAL; + + argp->type = V4L2_FRMIVAL_TYPE_DISCRETE; + argp->discrete = dev->capture_param.timeperframe; + } else { + if (argp->width < dev->min_width || + argp->width > dev->max_width || + argp->height < dev->min_height || + argp->height > dev->max_height || + !format_by_fourcc(argp->pixel_format)) + return -EINVAL; + + argp->type = V4L2_FRMIVAL_TYPE_CONTINUOUS; + argp->stepwise.min.numerator = 1; + argp->stepwise.min.denominator = V4L2LOOPBACK_FPS_MAX; + argp->stepwise.max.numerator = V4L2LOOPBACK_FRAME_INTERVAL_MAX; + argp->stepwise.max.denominator = 1; + argp->stepwise.step.numerator = 1; + argp->stepwise.step.denominator = 1; + } + + return 0; +} + +/* Enumerate device formats + * Returns: + * - EINVAL the index is out of bounds; or if non-zero when format is fixed + * - EFAULT unexpected null pointer */ +static int vidioc_enum_fmt_vid(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + int fixed = dev->keep_format || has_other_owners(opener, dev); + const struct v4l2l_format *fmt; + + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + + if (!(f->index < FORMATS)) + return -EINVAL; + /* TODO: Support 6.14 V4L2_FMTDESC_FLAG_ENUM_ALL */ + if (fixed && f->index) + return -EINVAL; + + fmt = fixed ? format_by_fourcc(dev->pix_format.pixelformat) : + &formats[f->index]; + if (!fmt) + return -EFAULT; + + f->flags = 0; + if (fmt->flags & FORMAT_FLAGS_COMPRESSED) + f->flags |= V4L2_FMT_FLAG_COMPRESSED; + snprintf(f->description, sizeof(f->description), fmt->name); + f->pixelformat = fmt->fourcc; + return 0; +} + +/* Tests (or tries) the format. + * Returns: + * - EINVAL if the buffer type or format is not supported + */ +static int vidioc_try_fmt_vid(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + if (v4l2l_fill_format(f, dev->min_width, dev->max_width, + dev->min_height, dev->max_height) != 0) + return -EINVAL; + if (dev->keep_format || has_other_owners(opener, dev)) + /* use existing format - including colorspace info */ + f->fmt.pix = dev->pix_format; + + return 0; +} + +/* Sets new format. Fills 'f' argument with the requested or existing format. + * Side-effect: buffers are allocated for the (returned) format. + * Returns: + * - EINVAL if the type is not supported + * - EBUSY if buffers are already allocated + * TODO: (vasaka) set subregions of input + */ +static int vidioc_s_fmt_vid(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = opener->io_method == V4L2L_IO_TIMEOUT ? + V4L2L_TOKEN_TIMEOUT : + token_from_type(f->type); + int changed, result; + char buf[5]; + + result = vidioc_try_fmt_vid(file, fh, f); + if (result < 0) + return result; + + if (opener->buffer_count > 0) + /* must free buffers before format can be set */ + return -EBUSY; + + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + + if (opener->format_token) + release_token(dev, opener, format); + if (!(dev->format_tokens & token)) { + result = -EBUSY; + goto exit_s_fmt_unlock; + } + + dprintk("S_FMT[%s] %4s:%ux%u size=%u\n", + V4L2_TYPE_IS_CAPTURE(f->type) ? "CAPTURE" : "OUTPUT", + fourcc2str(f->fmt.pix.pixelformat, buf), f->fmt.pix.width, + f->fmt.pix.height, f->fmt.pix.sizeimage); + changed = !pix_format_eq(&dev->pix_format, &f->fmt.pix, 0); + if (changed || has_no_owners(dev)) { + result = allocate_buffers(dev, &f->fmt.pix); + if (result < 0) + goto exit_s_fmt_unlock; + } + if ((dev->timeout_image && changed) || + (!dev->timeout_image && need_timeout_buffer(dev, token))) { + result = allocate_timeout_buffer(dev); + if (result < 0) + goto exit_s_fmt_free; + } + if (changed) { + dev->pix_format = f->fmt.pix; + dev->pix_format_has_valid_sizeimage = + v4l2l_pix_format_has_valid_sizeimage(f); + } + acquire_token(dev, opener, format, token); + if (opener->io_method == V4L2L_IO_TIMEOUT) + dev->timeout_image_io = 0; + goto exit_s_fmt_unlock; +exit_s_fmt_free: + free_buffers(dev); +exit_s_fmt_unlock: + mutex_unlock(&dev->image_mutex); + return result; +} + +/* ------------------ CAPTURE ----------------------- */ +/* ioctl for VIDIOC_ENUM_FMT, _G_FMT, _S_FMT, and _TRY_FMT when buffer type + * is V4L2_BUF_TYPE_VIDEO_CAPTURE */ + +static int vidioc_enum_fmt_cap(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt_vid(file, fh, f); +} + +static int vidioc_g_fmt_cap(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + f->fmt.pix = dev->pix_format; + return 0; +} + +static int vidioc_try_fmt_cap(struct file *file, void *fh, + struct v4l2_format *f) +{ + return vidioc_try_fmt_vid(file, fh, f); +} + +static int vidioc_s_fmt_cap(struct file *file, void *fh, struct v4l2_format *f) +{ + return vidioc_s_fmt_vid(file, fh, f); +} + +/* ------------------ OUTPUT ----------------------- */ +/* ioctl for VIDIOC_ENUM_FMT, _G_FMT, _S_FMT, and _TRY_FMT when buffer type + * is V4L2_BUF_TYPE_VIDEO_OUTPUT */ + +static int vidioc_enum_fmt_out(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return vidioc_enum_fmt_vid(file, fh, f); +} + +static int vidioc_g_fmt_out(struct file *file, void *fh, struct v4l2_format *f) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, f->type) < 0) + return -EINVAL; + /* + * LATER: this should return the currently valid format + * gstreamer doesn't like it, if this returns -EINVAL, as it + * then concludes that there is _no_ valid format + * CHECK whether this assumption is wrong, + * or whether we have to always provide a valid format + */ + f->fmt.pix = dev->pix_format; + return 0; +} + +static int vidioc_try_fmt_out(struct file *file, void *fh, + struct v4l2_format *f) +{ + return vidioc_try_fmt_vid(file, fh, f); +} + +static int vidioc_s_fmt_out(struct file *file, void *fh, struct v4l2_format *f) +{ + return vidioc_s_fmt_vid(file, fh, f); +} + +// #define V4L2L_OVERLAY +#ifdef V4L2L_OVERLAY +/* ------------------ OVERLAY ----------------------- */ +/* currently unsupported */ +/* GSTreamer's v4l2sink is buggy, as it requires the overlay to work + * while it should only require it, if overlay is requested + * once the gstreamer element is fixed, remove the overlay dummies + */ +#warning OVERLAY dummies +static int vidioc_g_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} + +static int vidioc_s_fmt_overlay(struct file *file, void *priv, + struct v4l2_format *fmt) +{ + return 0; +} +#endif /* V4L2L_OVERLAY */ + +/* ------------------ PARAMs ----------------------- */ + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_G_PARM + */ +static int vidioc_g_parm(struct file *file, void *fh, + struct v4l2_streamparm *parm) +{ + /* do not care about type of opener, hope these enums would always be + * compatible */ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, parm->type) < 0) + return -EINVAL; + parm->parm.capture = dev->capture_param; + return 0; +} + +/* get some data flow parameters, only capability, fps and readbuffers has + * effect on this driver + * called on VIDIOC_S_PARM + */ +static int vidioc_s_parm(struct file *file, void *fh, + struct v4l2_streamparm *parm) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + dprintk("S_PARM(frame-time=%u/%u)\n", + parm->parm.capture.timeperframe.numerator, + parm->parm.capture.timeperframe.denominator); + if (check_buffer_capability(dev, opener, parm->type) < 0) + return -EINVAL; + + switch (parm->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + set_timeperframe(dev, &parm->parm.capture.timeperframe); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + set_timeperframe(dev, &parm->parm.output.timeperframe); + break; + default: + return -EINVAL; + } + + parm->parm.capture = dev->capture_param; + return 0; +} + +#ifdef V4L2LOOPBACK_WITH_STD +/* sets a tv standard, actually we do not need to handle this any special way + * added to support effecttv + * called on VIDIOC_S_STD + */ +static int vidioc_s_std(struct file *file, void *fh, v4l2_std_id *_std) +{ + v4l2_std_id req_std = 0, supported_std = 0; + const v4l2_std_id all_std = V4L2_STD_ALL, no_std = 0; + + if (_std) { + req_std = *_std; + *_std = all_std; + } + + /* we support everything in V4L2_STD_ALL, but not more... */ + supported_std = (all_std & req_std); + if (no_std == supported_std) + return -EINVAL; + + return 0; +} + +/* gets a fake video standard + * called on VIDIOC_G_STD + */ +static int vidioc_g_std(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +/* gets a fake video standard + * called on VIDIOC_QUERYSTD + */ +static int vidioc_querystd(struct file *file, void *fh, v4l2_std_id *norm) +{ + if (norm) + *norm = V4L2_STD_ALL; + return 0; +} +#endif /* V4L2LOOPBACK_WITH_STD */ + +static int v4l2loopback_set_ctrl(struct v4l2_loopback_device *dev, u32 id, + s64 val) +{ + int result = 0; + switch (id) { + case CID_KEEP_FORMAT: + if (val < 0 || val > 1) + return -EINVAL; + dev->keep_format = val; + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + if (!dev->keep_format) { + if (has_no_owners(dev) && !any_buffers_mapped(dev)) + free_buffers(dev); + } + mutex_unlock(&dev->image_mutex); + break; + case CID_SUSTAIN_FRAMERATE: + if (val < 0 || val > 1) + return -EINVAL; + spin_lock_bh(&dev->lock); + dev->sustain_framerate = val; + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT: + if (val < 0 || val > MAX_TIMEOUT) + return -EINVAL; + if (val > 0) { + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + /* on-the-fly allocate if device is owned; else + * allocate occurs on next S_FMT or REQBUFS */ + if (!has_no_owners(dev)) + result = allocate_timeout_buffer(dev); + mutex_unlock(&dev->image_mutex); + if (result < 0) { + /* disable timeout as buffer not alloc'd */ + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = 0; + spin_unlock_bh(&dev->lock); + return result; + } + } + spin_lock_bh(&dev->lock); + dev->timeout_jiffies = msecs_to_jiffies(val); + check_timers(dev); + spin_unlock_bh(&dev->lock); + break; + case CID_TIMEOUT_IMAGE_IO: + dev->timeout_image_io = 1; + break; + default: + return -EINVAL; + } + return 0; +} + +static int v4l2loopback_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct v4l2_loopback_device *dev = container_of( + ctrl->handler, struct v4l2_loopback_device, ctrl_handler); + return v4l2loopback_set_ctrl(dev, ctrl->id, ctrl->val); +} + +/* returns set of device outputs, in our case there is only one + * called on VIDIOC_ENUMOUTPUT + */ +static int vidioc_enum_output(struct file *file, void *fh, + struct v4l2_output *outp) +{ + __u32 index = outp->index; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + if (index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(outp, 0, sizeof(*outp)); + + outp->index = index; + strscpy(outp->name, "loopback in", sizeof(outp->name)); + outp->type = V4L2_OUTPUT_TYPE_ANALOG; + outp->audioset = 0; + outp->modulator = 0; +#ifdef V4L2LOOPBACK_WITH_STD + outp->std = V4L2_STD_ALL; +#ifdef V4L2_OUT_CAP_STD + outp->capabilities |= V4L2_OUT_CAP_STD; +#endif /* V4L2_OUT_CAP_STD */ +#endif /* V4L2LOOPBACK_WITH_STD */ + + return 0; +} + +/* which output is currently active, + * called on VIDIOC_G_OUTPUT + */ +static int vidioc_g_output(struct file *file, void *fh, unsigned int *index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + if (index) + *index = 0; + return 0; +} + +/* set output, can make sense if we have more than one video src, + * called on VIDIOC_S_OUTPUT + */ +static int vidioc_s_output(struct file *file, void *fh, unsigned int index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -ENOTTY; + return index == 0 ? index : -EINVAL; +} + +/* returns set of device inputs, in our case there is only one, + * but later I may add more + * called on VIDIOC_ENUMINPUT + */ +static int vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + __u32 index = inp->index; + + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; + if (index) + return -EINVAL; + + /* clear all data (including the reserved fields) */ + memset(inp, 0, sizeof(*inp)); + + inp->index = index; + strscpy(inp->name, "loopback", sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + inp->audioset = 0; + inp->tuner = 0; + inp->status = 0; + +#ifdef V4L2LOOPBACK_WITH_STD + inp->std = V4L2_STD_ALL; +#ifdef V4L2_IN_CAP_STD + inp->capabilities |= V4L2_IN_CAP_STD; +#endif +#endif /* V4L2LOOPBACK_WITH_STD */ + + if (has_output_token(dev->stream_tokens) && !dev->keep_format) + /* if no outputs attached; pretend device is powered off */ + inp->status |= V4L2_IN_ST_NO_SIGNAL; + + return 0; +} + +/* which input is currently active, + * called on VIDIOC_G_INPUT + */ +static int vidioc_g_input(struct file *file, void *fh, unsigned int *index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; /* NOTE: -EAGAIN might be more informative */ + if (index) + *index = 0; + return 0; +} + +/* set input, can make sense if we have more than one video src, + * called on VIDIOC_S_INPUT + */ +static int vidioc_s_input(struct file *file, void *fh, unsigned int index) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + if (index != 0) + return -EINVAL; + if (check_buffer_capability(dev, opener, V4L2_BUF_TYPE_VIDEO_CAPTURE)) + return -ENOTTY; /* NOTE: -EAGAIN might be more informative */ + return 0; +} + +/* --------------- V4L2 ioctl buffer related calls ----------------- */ + +#define is_allocated(opener, type, index) \ + (opener->format_token & (opener->io_method == V4L2L_IO_TIMEOUT ? \ + V4L2L_TOKEN_TIMEOUT : \ + token_from_type(type)) && \ + (index) < (opener)->buffer_count) +#define BUFFER_DEBUG_FMT_STR \ + "buffer#%u @ %p type=%u bytesused=%u length=%u flags=%x " \ + "field=%u timestamp= %lld.%06lldsequence=%u\n" +#define BUFFER_DEBUG_FMT_ARGS(buf) \ + (buf)->index, (buf), (buf)->type, (buf)->bytesused, (buf)->length, \ + (buf)->flags, (buf)->field, \ + (long long)(buf)->timestamp.tv_sec, \ + (long long)(buf)->timestamp.tv_usec, (buf)->sequence +/* Buffer flag helpers */ +#define unset_flags(flags) \ + do { \ + flags &= ~V4L2_BUF_FLAG_QUEUED; \ + flags &= ~V4L2_BUF_FLAG_DONE; \ + } while (0) +#define set_queued(flags) \ + do { \ + flags |= V4L2_BUF_FLAG_QUEUED; \ + flags &= ~V4L2_BUF_FLAG_DONE; \ + } while (0) +#define set_done(flags) \ + do { \ + flags &= ~V4L2_BUF_FLAG_QUEUED; \ + flags |= V4L2_BUF_FLAG_DONE; \ + } while (0) + +static bool any_buffers_mapped(struct v4l2_loopback_device *dev) +{ + u32 index; + for (index = 0; index < dev->buffer_count; ++index) + if (dev->buffers[index].buffer.flags & V4L2_BUF_FLAG_MAPPED) + return true; + return false; +} + +static void prepare_buffer_queue(struct v4l2_loopback_device *dev, int count) +{ + struct v4l2l_buffer *bufd, *n; + u32 pos; + + spin_lock_bh(&dev->list_lock); + + /* ensure sufficient number of buffers in queue */ + for (pos = 0; pos < count; ++pos) { + bufd = &dev->buffers[pos]; + if (list_empty(&bufd->list_head)) + list_add_tail(&bufd->list_head, &dev->outbufs_list); + } + if (list_empty(&dev->outbufs_list)) + goto exit_prepare_queue_unlock; + + /* remove any excess buffers */ + list_for_each_entry_safe(bufd, n, &dev->outbufs_list, list_head) { + if (bufd->buffer.index >= count) + list_del_init(&bufd->list_head); + } + + /* buffers are no longer queued; and `write_position` will correspond + * to the first item of `outbufs_list`. */ + pos = v4l2l_mod64(dev->write_position, count); + list_for_each_entry(bufd, &dev->outbufs_list, list_head) { + unset_flags(bufd->buffer.flags); + dev->bufpos2index[pos % count] = bufd->buffer.index; + ++pos; + } +exit_prepare_queue_unlock: + spin_unlock_bh(&dev->list_lock); +} + +/* forward declaration */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type); +/* negotiate buffer type + * only mmap streaming supported + * called on VIDIOC_REQBUFS + */ +static int vidioc_reqbufs(struct file *file, void *fh, + struct v4l2_requestbuffers *reqbuf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = opener->io_method == V4L2L_IO_TIMEOUT ? + V4L2L_TOKEN_TIMEOUT : + token_from_type(reqbuf->type); + u32 req_count = reqbuf->count; + int result = 0; + + dprintk("REQBUFS(memory=%u, req_count=%u) and device-bufs=%u/%u " + "[used/max]\n", + reqbuf->memory, req_count, dev->used_buffer_count, + dev->buffer_count); + + switch (reqbuf->memory) { + case V4L2_MEMORY_MMAP: +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) + reqbuf->capabilities = 0; /* only guarantee MMAP support */ +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + reqbuf->flags = 0; /* no memory consistency support */ +#endif + break; + default: + return -EINVAL; + } + + if (opener->format_token & ~token) + /* different (buffer) type already assigned to descriptor by + * S_FMT or REQBUFS */ + return -EINVAL; + + MARK(); + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; /* -EINTR */ + + /* CASE queue/dequeue timeout-buffer only: */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + opener->buffer_count = req_count; + if (req_count == 0) + release_token(dev, opener, format); + goto exit_reqbufs_unlock; + } + + MARK(); + /* CASE count is zero: streamoff, free buffers, release their token */ + if (req_count == 0) { + if (dev->format_tokens & token) { + acquire_token(dev, opener, format, token); + opener->io_method = V4L2L_IO_MMAP; + } + result = vidioc_streamoff(file, fh, reqbuf->type); + opener->buffer_count = 0; + /* undocumented requirement - REQBUFS with count zero should + * ALSO release lock on logical stream */ + if (opener->format_token) + release_token(dev, opener, format); + if (has_no_owners(dev)) + dev->used_buffer_count = 0; + goto exit_reqbufs_unlock; + } + + /* CASE count non-zero: allocate buffers and acquire token for them */ + MARK(); + switch (reqbuf->type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (!(dev->format_tokens & token || + opener->format_token & token)) + /* only exclusive ownership for each stream */ + result = -EBUSY; + break; + default: + result = -EINVAL; + } + if (result < 0) + goto exit_reqbufs_unlock; + + if (has_other_owners(opener, dev) && dev->used_buffer_count > 0) { + /* allow 'allocation' of existing number of buffers */ + req_count = dev->used_buffer_count; + } else if (any_buffers_mapped(dev)) { + /* do not allow re-allocation if buffers are mapped */ + result = -EBUSY; + goto exit_reqbufs_unlock; + } + + MARK(); + opener->buffer_count = 0; + + if (req_count > dev->buffer_count) + req_count = dev->buffer_count; + + if (has_no_owners(dev)) { + result = allocate_buffers(dev, &dev->pix_format); + if (result < 0) + goto exit_reqbufs_unlock; + } + if (!dev->timeout_image && need_timeout_buffer(dev, token)) { + result = allocate_timeout_buffer(dev); + if (result < 0) + goto exit_reqbufs_unlock; + } + acquire_token(dev, opener, format, token); + + MARK(); + switch (opener->io_method) { + case V4L2L_IO_TIMEOUT: + dev->timeout_image_io = 0; + opener->buffer_count = req_count; + break; + default: + opener->io_method = V4L2L_IO_MMAP; + prepare_buffer_queue(dev, req_count); + dev->used_buffer_count = opener->buffer_count = req_count; + } +exit_reqbufs_unlock: + mutex_unlock(&dev->image_mutex); + reqbuf->count = opener->buffer_count; + return result; +} + +/* returns buffer asked for; + * give app as many buffers as it wants, if it less than MAX, + * but map them in our inner buffers + * called on VIDIOC_QUERYBUF + */ +static int vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 type = buf->type; + u32 index = buf->index; + + if ((type != V4L2_BUF_TYPE_VIDEO_CAPTURE) && + (type != V4L2_BUF_TYPE_VIDEO_OUTPUT)) + return -EINVAL; + if (!is_allocated(opener, type, index)) + return -EINVAL; + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + *buf = dev->timeout_buffer.buffer; + buf->index = index; + } else + *buf = dev->buffers[index].buffer; + + buf->type = type; + + if (!(buf->flags & (V4L2_BUF_FLAG_DONE | V4L2_BUF_FLAG_QUEUED))) { + /* v4l2-compliance requires these to be zero */ + buf->sequence = 0; + buf->timestamp.tv_sec = buf->timestamp.tv_usec = 0; + } else if (V4L2_TYPE_IS_CAPTURE(type)) { + /* guess flags based on sequence values */ + if (buf->sequence >= opener->read_position) { + set_done(buf->flags); + } else if (buf->flags & V4L2_BUF_FLAG_DONE) { + set_queued(buf->flags); + } + } + dprintkrw("QUERYBUF(%s, index=%u) -> " BUFFER_DEBUG_FMT_STR, + V4L2_TYPE_IS_CAPTURE(type) ? "CAPTURE" : "OUTPUT", index, + BUFFER_DEBUG_FMT_ARGS(buf)); + return 0; +} + +static void buffer_written(struct v4l2_loopback_device *dev, + struct v4l2l_buffer *buf) +{ + timer_delete_sync(&dev->sustain_timer); + timer_delete_sync(&dev->timeout_timer); + + spin_lock_bh(&dev->list_lock); + list_move_tail(&buf->list_head, &dev->outbufs_list); + spin_unlock_bh(&dev->list_lock); + + spin_lock_bh(&dev->lock); + dev->bufpos2index[v4l2l_mod64(dev->write_position, + dev->used_buffer_count)] = + buf->buffer.index; + ++dev->write_position; + dev->reread_count = 0; + + check_timers(dev); + spin_unlock_bh(&dev->lock); +} + +/* put buffer to queue + * called on VIDIOC_QBUF + */ +static int vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + struct v4l2l_buffer *bufd; + u32 index = buf->index; + u32 type = buf->type; + + if (!is_allocated(opener, type, index)) + return -EINVAL; + bufd = &dev->buffers[index]; + + switch (buf->memory) { + case V4L2_MEMORY_MMAP: + if (!(bufd->buffer.flags & V4L2_BUF_FLAG_MAPPED)) + dprintkrw("QBUF() unmapped buffer [index=%u]\n", index); + break; + default: + return -EINVAL; + } + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + set_queued(buf->flags); + return 0; + } + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + dprintkrw("QBUF(CAPTURE, index=%u) -> " BUFFER_DEBUG_FMT_STR, + index, BUFFER_DEBUG_FMT_ARGS(buf)); + set_queued(buf->flags); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + dprintkrw("QBUF(OUTPUT, index=%u) -> " BUFFER_DEBUG_FMT_STR, + index, BUFFER_DEBUG_FMT_ARGS(buf)); + if (!(bufd->buffer.flags & V4L2_BUF_FLAG_TIMESTAMP_COPY) && + (buf->timestamp.tv_sec == 0 && + buf->timestamp.tv_usec == 0)) { + v4l2l_get_timestamp(&bufd->buffer); + } else { + bufd->buffer.timestamp = buf->timestamp; + bufd->buffer.flags |= V4L2_BUF_FLAG_TIMESTAMP_COPY; + bufd->buffer.flags &= + ~V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + } + if (dev->pix_format_has_valid_sizeimage) { + if (buf->bytesused >= dev->pix_format.sizeimage) { + bufd->buffer.bytesused = + dev->pix_format.sizeimage; + } else { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) + dev_warn_ratelimited( + &dev->vdev->dev, +#else + dprintkrw( +#endif + "warning queued output buffer bytesused too small %u < %u\n", + buf->bytesused, + dev->pix_format.sizeimage); + bufd->buffer.bytesused = buf->bytesused; + } + } else { + bufd->buffer.bytesused = buf->bytesused; + } + bufd->buffer.sequence = dev->write_position; + set_queued(bufd->buffer.flags); + *buf = bufd->buffer; + buffer_written(dev, bufd); + set_done(bufd->buffer.flags); + wake_up_all(&dev->read_event); + break; + default: + return -EINVAL; + } + buf->type = type; + return 0; +} + +static int can_read(struct v4l2_loopback_device *dev, + struct v4l2_loopback_opener *opener) +{ + int ret; + + spin_lock_bh(&dev->lock); + check_timers(dev); + ret = dev->write_position > opener->read_position || + dev->reread_count > opener->reread_count || dev->timeout_happened; + spin_unlock_bh(&dev->lock); + return ret; +} + +static int get_capture_buffer(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + int pos, timeout_happened; + u32 index; + + if ((file->f_flags & O_NONBLOCK) && + (dev->write_position <= opener->read_position && + dev->reread_count <= opener->reread_count && + !dev->timeout_happened)) + return -EAGAIN; + wait_event_interruptible(dev->read_event, can_read(dev, opener)); + + spin_lock_bh(&dev->lock); + if (dev->write_position == opener->read_position) { + if (dev->reread_count > opener->reread_count + 2) + opener->reread_count = dev->reread_count - 1; + ++opener->reread_count; + pos = v4l2l_mod64(opener->read_position + + dev->used_buffer_count - 1, + dev->used_buffer_count); + } else { + opener->reread_count = 0; + if (dev->write_position > + opener->read_position + dev->used_buffer_count) + opener->read_position = dev->write_position - 1; + pos = v4l2l_mod64(opener->read_position, + dev->used_buffer_count); + ++opener->read_position; + } + timeout_happened = dev->timeout_happened && (dev->timeout_jiffies > 0); + dev->timeout_happened = 0; + spin_unlock_bh(&dev->lock); + + index = dev->bufpos2index[pos]; + if (timeout_happened) { + if (index >= dev->used_buffer_count) { + dprintkrw("get_capture_buffer() read position is at " + "an unallocated buffer [index=%u]\n", + index); + return -EFAULT; + } + /* although allocated on-demand, timeout_image is freed only + * in free_buffers(), so we don't need to worry about it being + * deallocated suddenly */ + memcpy(dev->image + dev->buffers[index].buffer.m.offset, + dev->timeout_image, dev->buffer_size); + } + return (int)index; +} + +/* put buffer to dequeue + * called on VIDIOC_DQBUF + */ +static int vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 type = buf->type; + int index; + struct v4l2l_buffer *bufd; + + if (buf->memory != V4L2_MEMORY_MMAP) + return -EINVAL; + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) { + *buf = dev->timeout_buffer.buffer; + buf->type = type; + unset_flags(buf->flags); + return 0; + } + if ((opener->buffer_count == 0) || + !(opener->format_token & token_from_type(type))) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + index = get_capture_buffer(file); + if (index < 0) + return index; + *buf = dev->buffers[index].buffer; + unset_flags(buf->flags); + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + spin_lock_bh(&dev->list_lock); + + bufd = list_first_entry_or_null(&dev->outbufs_list, + struct v4l2l_buffer, list_head); + if (bufd) + list_move_tail(&bufd->list_head, &dev->outbufs_list); + + spin_unlock_bh(&dev->list_lock); + if (!bufd) + return -EFAULT; + unset_flags(bufd->buffer.flags); + *buf = bufd->buffer; + break; + default: + return -EINVAL; + } + + buf->type = type; + dprintkrw("DQBUF(%s, index=%u) -> " BUFFER_DEBUG_FMT_STR, + V4L2_TYPE_IS_CAPTURE(type) ? "CAPTURE" : "OUTPUT", index, + BUFFER_DEBUG_FMT_ARGS(buf)); + return 0; +} + +/* ------------- STREAMING ------------------- */ + +/* start streaming + * called on VIDIOC_STREAMON + */ +static int vidioc_streamon(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = token_from_type(type); + + /* short-circuit when using timeout buffer set */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) + return 0; + /* opener must have claimed (same) buffer set via REQBUFS */ + if (!opener->buffer_count || !(opener->format_token & token)) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (has_output_token(dev->stream_tokens) && !dev->keep_format) + return -EIO; + if (dev->stream_tokens & token) { + acquire_token(dev, opener, stream, token); + client_usage_queue_event(dev->vdev); + } + return 0; + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (dev->stream_tokens & token) + acquire_token(dev, opener, stream, token); + return 0; + default: + return -EINVAL; + } +} + +/* stop streaming + * called on VIDIOC_STREAMOFF + */ +static int vidioc_streamoff(struct file *file, void *fh, + enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + u32 token = token_from_type(type); + + /* short-circuit when using timeout buffer set */ + if (opener->format_token & V4L2L_TOKEN_TIMEOUT) + return 0; + /* short-circuit when buffer set has no owner */ + if (dev->format_tokens & token) + return 0; + /* opener needs a claim to buffer set */ + if (!opener->format_token) + return -EBUSY; + if (opener->format_token & ~token) + return -EINVAL; + + switch (type) { + case V4L2_BUF_TYPE_VIDEO_OUTPUT: + if (opener->stream_token & token) + release_token(dev, opener, stream); + /* reset output queue */ + if (dev->used_buffer_count > 0) + prepare_buffer_queue(dev, dev->used_buffer_count); + return 0; + case V4L2_BUF_TYPE_VIDEO_CAPTURE: + if (opener->stream_token & token) { + release_token(dev, opener, stream); + client_usage_queue_event(dev->vdev); + } + return 0; + default: + return -EINVAL; + } +} + +#ifdef CONFIG_VIDEO_V4L1_COMPAT +static int vidiocgmbuf(struct file *file, void *fh, struct video_mbuf *p) +{ + struct v4l2_loopback_device *dev; + MARK(); + + dev = v4l2loopback_getdevice(file); + p->frames = dev->buffer_count; + p->offsets[0] = 0; + p->offsets[1] = 0; + p->size = dev->buffer_size; + return 0; +} +#endif + +static void client_usage_queue_event(struct video_device *vdev) +{ + struct v4l2_event ev; + struct v4l2_loopback_device *dev; + + dev = container_of(vdev->v4l2_dev, struct v4l2_loopback_device, + v4l2_dev); + + memset(&ev, 0, sizeof(ev)); + ev.type = V4L2_EVENT_PRI_CLIENT_USAGE; + ((struct v4l2_event_client_usage *)&ev.u)->count = + !has_capture_token(dev->stream_tokens); + + v4l2_event_queue(vdev, &ev); +} + +static int client_usage_ops_add(struct v4l2_subscribed_event *sev, + unsigned elems) +{ + if (!(sev->flags & V4L2_EVENT_SUB_FL_SEND_INITIAL)) + return 0; + + client_usage_queue_event(sev->fh->vdev); + return 0; +} + +static void client_usage_ops_replace(struct v4l2_event *old, + const struct v4l2_event *new) +{ + *((struct v4l2_event_client_usage *)&old->u) = + *((struct v4l2_event_client_usage *)&new->u); +} + +static void client_usage_ops_merge(const struct v4l2_event *old, + struct v4l2_event *new) +{ + *((struct v4l2_event_client_usage *)&new->u) = + *((struct v4l2_event_client_usage *)&old->u); +} + +const struct v4l2_subscribed_event_ops client_usage_ops = { + .add = client_usage_ops_add, + .replace = client_usage_ops_replace, + .merge = client_usage_ops_merge, +}; + +static int vidioc_subscribe_event(struct v4l2_fh *fh, + const struct v4l2_event_subscription *sub) +{ + switch (sub->type) { + case V4L2_EVENT_CTRL: + return v4l2_ctrl_subscribe_event(fh, sub); + case V4L2_EVENT_PRI_CLIENT_USAGE: + return v4l2_event_subscribe(fh, sub, 0, &client_usage_ops); + } + + return -EINVAL; +} + +/* file operations */ +static void vm_open(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf; + MARK(); + + buf = vma->vm_private_data; + atomic_inc(&buf->use_count); + buf->buffer.flags |= V4L2_BUF_FLAG_MAPPED; +} + +static void vm_close(struct vm_area_struct *vma) +{ + struct v4l2l_buffer *buf; + MARK(); + + buf = vma->vm_private_data; + if (atomic_dec_and_test(&buf->use_count)) + buf->buffer.flags &= ~V4L2_BUF_FLAG_MAPPED; +} + +static struct vm_operations_struct vm_ops = { + .open = vm_open, + .close = vm_close, +}; + +static int v4l2_loopback_mmap(struct file *file, struct vm_area_struct *vma) +{ + u8 *addr; + unsigned long start, size, offset; + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + struct v4l2l_buffer *buffer = NULL; + int result = 0; + MARK(); + + offset = (unsigned long)vma->vm_pgoff << PAGE_SHIFT; + start = (unsigned long)vma->vm_start; + size = (unsigned long)(vma->vm_end - vma->vm_start); /* always != 0 */ + + /* ensure buffer size, count, and allocated image(s) are not altered by + * other file descriptors */ + result = mutex_lock_killable(&dev->image_mutex); + if (result < 0) + return result; + + if (size > dev->buffer_size) { + dprintk("mmap() attempt to map %lubytes when %ubytes are " + "allocated to buffers\n", + size, dev->buffer_size); + result = -EINVAL; + goto exit_mmap_unlock; + } + if (offset % dev->buffer_size != 0) { + dprintk("mmap() offset does not match start of any buffer\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + switch (opener->format_token) { + case V4L2L_TOKEN_TIMEOUT: + if (offset != (unsigned long)dev->buffer_size * MAX_BUFFERS) { + dprintk("mmap() incorrect offset for timeout image\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + buffer = &dev->timeout_buffer; + addr = dev->timeout_image; + break; + default: + if (offset >= dev->image_size) { + dprintk("mmap() attempt to map beyond all buffers\n"); + result = -EINVAL; + goto exit_mmap_unlock; + } + u32 index = offset / dev->buffer_size; + buffer = &dev->buffers[index]; + addr = dev->image + offset; + break; + } + + while (size > 0) { + struct page *page = vmalloc_to_page(addr); + + result = vm_insert_page(vma, start, page); + if (result < 0) + goto exit_mmap_unlock; + + start += PAGE_SIZE; + addr += PAGE_SIZE; + size -= PAGE_SIZE; + } + + vma->vm_ops = &vm_ops; + vma->vm_private_data = buffer; + + vm_open(vma); +exit_mmap_unlock: + mutex_unlock(&dev->image_mutex); + return result; +} + +static unsigned int v4l2_loopback_poll(struct file *file, + struct poll_table_struct *pts) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + __poll_t req_events = poll_requested_events(pts); + int ret_mask = 0; + + /* call poll_wait in first call, regardless, to ensure that the + * wait-queue is not null */ + poll_wait(file, &dev->read_event, pts); + poll_wait(file, &opener->fh.wait, pts); + + if (req_events & POLLPRI) { + if (v4l2_event_pending(&opener->fh)) { + ret_mask |= POLLPRI; + if (!(req_events & DEFAULT_POLLMASK)) + return ret_mask; + } + } + + switch (opener->format_token) { + case V4L2L_TOKEN_OUTPUT: + if (opener->stream_token != 0 || + opener->io_method == V4L2L_IO_NONE) + ret_mask |= POLLOUT | POLLWRNORM; + break; + case V4L2L_TOKEN_CAPTURE: + if ((opener->io_method == V4L2L_IO_NONE || + opener->stream_token != 0) && + can_read(dev, opener)) + ret_mask |= POLLIN | POLLWRNORM; + break; + case V4L2L_TOKEN_TIMEOUT: + ret_mask |= POLLOUT | POLLWRNORM; + break; + default: + break; + } + + return ret_mask; +} + +/* do not want to limit device opens, it can be as many readers as user want, + * writers are limited by means of setting writer field */ +static int v4l2_loopback_open(struct file *file) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_opener *opener; + + dev = v4l2loopback_getdevice(file); + if (dev->open_count.counter >= dev->max_openers) + return -EBUSY; + /* kfree on close */ + opener = kzalloc(sizeof(*opener), GFP_KERNEL); + if (opener == NULL) + return -ENOMEM; + + atomic_inc(&dev->open_count); + if (dev->timeout_image_io && dev->format_tokens & V4L2L_TOKEN_TIMEOUT) + /* will clear timeout_image_io once buffer set acquired */ + opener->io_method = V4L2L_IO_TIMEOUT; + + v4l2_fh_init(&opener->fh, video_devdata(file)); + file->private_data = &opener->fh; + + v4l2_fh_add(&opener->fh); + dprintk("open() -> dev@%p with image@%p\n", dev, + dev ? dev->image : NULL); + return 0; +} + +static int v4l2_loopback_close(struct file *file) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(file->private_data); + int result = 0; + dprintk("close() -> dev@%p with image@%p\n", dev, + dev ? dev->image : NULL); + + if (opener->format_token) { + struct v4l2_requestbuffers reqbuf = { + .count = 0, .memory = V4L2_MEMORY_MMAP, .type = 0 + }; + switch (opener->format_token) { + case V4L2L_TOKEN_CAPTURE: + reqbuf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + break; + case V4L2L_TOKEN_OUTPUT: + case V4L2L_TOKEN_TIMEOUT: + reqbuf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + break; + } + if (reqbuf.type) + result = vidioc_reqbufs(file, file->private_data, + &reqbuf); + if (result < 0) + dprintk("failed to free buffers REQBUFS(count=0) " + " returned %d\n", + result); + mutex_lock(&dev->image_mutex); + release_token(dev, opener, format); + mutex_unlock(&dev->image_mutex); + } + + if (atomic_dec_and_test(&dev->open_count)) { + timer_delete_sync(&dev->sustain_timer); + timer_delete_sync(&dev->timeout_timer); + if (!dev->keep_format) { + mutex_lock(&dev->image_mutex); + free_buffers(dev); + mutex_unlock(&dev->image_mutex); + } + } + + v4l2_fh_del(&opener->fh); + v4l2_fh_exit(&opener->fh); + + kfree(opener); + return 0; +} + +static int start_fileio(struct file *file, void *fh, enum v4l2_buf_type type) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_loopback_opener *opener = fh_to_opener(fh); + struct v4l2_requestbuffers reqbuf = { .count = dev->buffer_count, + .memory = V4L2_MEMORY_MMAP, + .type = type }; + int token = token_from_type(type); + int result; + + if (opener->format_token & V4L2L_TOKEN_TIMEOUT || + opener->format_token & ~token) + return -EBUSY; /* NOTE: -EBADF might be more informative */ + + /* short-circuit if already have stream token */ + if (opener->stream_token && opener->io_method == V4L2L_IO_FILE) + return 0; + + /* otherwise attempt to acquire stream token and assign IO method */ + if (!(dev->stream_tokens & token) || opener->io_method != V4L2L_IO_NONE) + return -EBUSY; + + result = vidioc_reqbufs(file, fh, &reqbuf); + if (result < 0) + return result; + result = vidioc_streamon(file, fh, type); + if (result < 0) + return result; + + opener->io_method = V4L2L_IO_FILE; + return 0; +} + +static ssize_t v4l2_loopback_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_buffer *b; + int index, result; + + dprintkrw("read() %zu bytes\n", count); + result = start_fileio(file, file->private_data, + V4L2_BUF_TYPE_VIDEO_CAPTURE); + if (result < 0) + return result; + + index = get_capture_buffer(file); + if (index < 0) + return index; + b = &dev->buffers[index].buffer; + if (count > b->bytesused) + count = b->bytesused; + if (copy_to_user((void *)buf, (void *)(dev->image + b->m.offset), + count)) { + printk(KERN_ERR "v4l2-loopback read() failed copy_to_user()\n"); + return -EFAULT; + } + return count; +} + +static ssize_t v4l2_loopback_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct v4l2_loopback_device *dev = v4l2loopback_getdevice(file); + struct v4l2_buffer *b; + int index, result; + + dprintkrw("write() %zu bytes\n", count); + result = start_fileio(file, file->private_data, + V4L2_BUF_TYPE_VIDEO_OUTPUT); + if (result < 0) + return result; + + if (count > dev->buffer_size) + count = dev->buffer_size; + index = v4l2l_mod64(dev->write_position, dev->used_buffer_count); + b = &dev->buffers[index].buffer; + + if (copy_from_user((void *)(dev->image + b->m.offset), (void *)buf, + count)) { + printk(KERN_ERR + "v4l2-loopback write() failed copy_from_user()\n"); + return -EFAULT; + } + b->bytesused = count; + + v4l2l_get_timestamp(b); + b->sequence = dev->write_position; + set_queued(b->flags); + buffer_written(dev, &dev->buffers[index]); + set_done(b->flags); + wake_up_all(&dev->read_event); + + return count; +} + +/* init functions */ +/* frees buffers, if allocated */ +static void free_buffers(struct v4l2_loopback_device *dev) +{ + dprintk("free_buffers() with image@%p\n", dev->image); + if (!dev->image) + return; + if (!has_no_owners(dev) || any_buffers_mapped(dev)) + /* maybe an opener snuck in before image_mutex was acquired */ + printk(KERN_WARNING + "v4l2-loopback free_buffers() buffers of video device " + "#%u freed while still mapped to userspace\n", + dev->vdev->num); + vfree(dev->image); + dev->image = NULL; + dev->image_size = 0; + dev->buffer_size = 0; +} + +static void free_timeout_buffer(struct v4l2_loopback_device *dev) +{ + dprintk("free_timeout_buffer() with timeout_image@%p\n", + dev->timeout_image); + if (!dev->timeout_image) + return; + + if ((dev->timeout_jiffies > 0 && !has_no_owners(dev)) || + dev->timeout_buffer.buffer.flags & V4L2_BUF_FLAG_MAPPED) + printk(KERN_WARNING + "v4l2-loopback free_timeout_buffer() timeout image " + "of device #%u freed while still mapped to userspace\n", + dev->vdev->num); + + vfree(dev->timeout_image); + dev->timeout_image = NULL; + dev->timeout_buffer_size = 0; +} +/* allocates buffers if no (other) openers are already using them */ +static int allocate_buffers(struct v4l2_loopback_device *dev, + struct v4l2_pix_format *pix_format) +{ + u32 buffer_size = PAGE_ALIGN(pix_format->sizeimage); + unsigned long image_size = + (unsigned long)buffer_size * (unsigned long)dev->buffer_count; + /* vfree on close file operation in case no open handles left */ + + if (buffer_size == 0 || dev->buffer_count == 0 || + buffer_size < pix_format->sizeimage) + return -EINVAL; + + if ((__LONG_MAX__ / buffer_size) < dev->buffer_count) + return -ENOSPC; + + dprintk("allocate_buffers() size %lubytes = %ubytes x %ubuffers\n", + image_size, buffer_size, dev->buffer_count); + if (dev->image) { + /* check that no buffers are expected in user-space */ + if (!has_no_owners(dev) || any_buffers_mapped(dev)) + return -EBUSY; + dprintk("allocate_buffers() existing size=%lubytes\n", + dev->image_size); + /* FIXME: prevent double allocation more intelligently! */ + if (image_size == dev->image_size) { + dprintk("allocate_buffers() keep existing\n"); + return 0; + } + free_buffers(dev); + } + + /* FIXME: set buffers to 0 */ + dev->image = vmalloc(image_size); + if (dev->image == NULL) { + dev->buffer_size = dev->image_size = 0; + return -ENOMEM; + } + init_buffers(dev, pix_format->sizeimage, buffer_size); + dev->buffer_size = buffer_size; + dev->image_size = image_size; + dprintk("allocate_buffers() -> vmalloc'd %lubytes\n", dev->image_size); + return 0; +} +static int allocate_timeout_buffer(struct v4l2_loopback_device *dev) +{ + /* device's `buffer_size` and `buffers` must be initialised in + * allocate_buffers() */ + + dprintk("allocate_timeout_buffer() size %ubytes\n", dev->buffer_size); + if (dev->buffer_size == 0) + return -EINVAL; + + if (dev->timeout_image) { + if (dev->timeout_buffer.buffer.flags & V4L2_BUF_FLAG_MAPPED) + return -EBUSY; + if (dev->buffer_size == dev->timeout_buffer_size) + return 0; + free_timeout_buffer(dev); + } + + dev->timeout_image = vzalloc(dev->buffer_size); + if (!dev->timeout_image) { + dev->timeout_buffer_size = 0; + return -ENOMEM; + } + dev->timeout_buffer_size = dev->buffer_size; + return 0; +} +/* init inner buffers, they are capture mode and flags are set as for capture + * mode buffers */ +static void init_buffers(struct v4l2_loopback_device *dev, u32 bytes_used, + u32 buffer_size) +{ + u32 i; + + for (i = 0; i < dev->buffer_count; ++i) { + struct v4l2_buffer *b = &dev->buffers[i].buffer; + b->index = i; + b->bytesused = bytes_used; + b->length = buffer_size; + b->field = V4L2_FIELD_NONE; + b->flags = 0; + b->m.offset = i * buffer_size; + b->memory = V4L2_MEMORY_MMAP; + b->sequence = 0; + b->timestamp.tv_sec = 0; + b->timestamp.tv_usec = 0; + b->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + v4l2l_get_timestamp(b); + } + dev->timeout_buffer = dev->buffers[0]; + dev->timeout_buffer.buffer.m.offset = MAX_BUFFERS * buffer_size; +} + +/* fills and register video device */ +static void init_vdev(struct video_device *vdev, int nr) +{ +#ifdef V4L2LOOPBACK_WITH_STD + vdev->tvnorms = V4L2_STD_ALL; +#endif /* V4L2LOOPBACK_WITH_STD */ + + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->fops = &v4l2_loopback_fops; + vdev->ioctl_ops = &v4l2_loopback_ioctl_ops; + vdev->release = &video_device_release; + vdev->minor = -1; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) + vdev->device_caps = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_VIDEO_CAPTURE | + V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_READWRITE | + V4L2_CAP_STREAMING; +#endif + + if (debug > 1) + vdev->dev_debug = V4L2_DEV_DEBUG_IOCTL | + V4L2_DEV_DEBUG_IOCTL_ARG; + + vdev->vfl_dir = VFL_DIR_M2M; +} + +/* init default capture parameters, only fps may be changed in future */ +static void init_capture_param(struct v4l2_captureparm *capture_param) +{ + capture_param->capability = V4L2_CAP_TIMEPERFRAME; /* since 2.16 */ + capture_param->capturemode = 0; + capture_param->extendedmode = 0; + capture_param->readbuffers = max_buffers; + capture_param->timeperframe.numerator = 1; + capture_param->timeperframe.denominator = V4L2LOOPBACK_FPS_DEFAULT; +} + +static void check_timers(struct v4l2_loopback_device *dev) +{ + if (has_output_token(dev->stream_tokens)) + return; + + if (dev->timeout_jiffies > 0 && !timer_pending(&dev->timeout_timer)) + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + if (dev->sustain_framerate && !timer_pending(&dev->sustain_timer)) + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies * 3 / 2); +} +#ifdef HAVE_TIMER_SETUP +static void sustain_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = from_timer(dev, t, sustain_timer); +#else +static void sustain_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->sustain_framerate) { + dev->reread_count++; + dprintkrw("sustain_timer_clb() write_pos=%lld reread=%u\n", + (long long)dev->write_position, dev->reread_count); + if (dev->reread_count == 1) + mod_timer(&dev->sustain_timer, + jiffies + max(1UL, dev->frame_jiffies / 2)); + else + mod_timer(&dev->sustain_timer, + jiffies + dev->frame_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} +#ifdef HAVE_TIMER_SETUP +static void timeout_timer_clb(struct timer_list *t) +{ + struct v4l2_loopback_device *dev = from_timer(dev, t, timeout_timer); +#else +static void timeout_timer_clb(unsigned long nr) +{ + struct v4l2_loopback_device *dev = + idr_find(&v4l2loopback_index_idr, nr); +#endif + spin_lock(&dev->lock); + if (dev->timeout_jiffies > 0) { + dev->timeout_happened = 1; + mod_timer(&dev->timeout_timer, jiffies + dev->timeout_jiffies); + wake_up_all(&dev->read_event); + } + spin_unlock(&dev->lock); +} + +/* init loopback main structure */ +#define DEFAULT_FROM_CONF(confmember, default_condition, default_value) \ + ((conf) ? \ + ((conf->confmember default_condition) ? (default_value) : \ + (conf->confmember)) : \ + default_value) + +static int v4l2_loopback_add(struct v4l2_loopback_config *conf, int *ret_nr) +{ + struct v4l2_loopback_device *dev; + struct v4l2_ctrl_handler *hdl; + struct v4l2loopback_private *vdev_priv = NULL; + int err; + + u32 _width = V4L2LOOPBACK_SIZE_DEFAULT_WIDTH; + u32 _height = V4L2LOOPBACK_SIZE_DEFAULT_HEIGHT; + + u32 _min_width = DEFAULT_FROM_CONF(min_width, + < V4L2LOOPBACK_SIZE_MIN_WIDTH, + V4L2LOOPBACK_SIZE_MIN_WIDTH); + u32 _min_height = DEFAULT_FROM_CONF(min_height, + < V4L2LOOPBACK_SIZE_MIN_HEIGHT, + V4L2LOOPBACK_SIZE_MIN_HEIGHT); + u32 _max_width = DEFAULT_FROM_CONF(max_width, < _min_width, max_width); + u32 _max_height = + DEFAULT_FROM_CONF(max_height, < _min_height, max_height); + bool _announce_all_caps = (conf && conf->announce_all_caps >= 0) ? + (bool)(conf->announce_all_caps) : + !(V4L2LOOPBACK_DEFAULT_EXCLUSIVECAPS); + int _max_buffers = DEFAULT_FROM_CONF(max_buffers, <= 0, max_buffers); + int _max_openers = DEFAULT_FROM_CONF(max_openers, <= 0, max_openers); + struct v4l2_format _fmt; + + int nr = -1; + + if (conf) { + const int output_nr = conf->output_nr; +#ifdef SPLIT_DEVICES + const int capture_nr = conf->capture_nr; +#else + const int capture_nr = output_nr; +#endif + if (capture_nr >= 0 && output_nr == capture_nr) { + nr = output_nr; + } else if (capture_nr < 0 && output_nr < 0) { + nr = -1; + } else if (capture_nr < 0) { + nr = output_nr; + } else if (output_nr < 0) { + nr = capture_nr; + } else { + printk(KERN_ERR + "v4l2-loopback add() split OUTPUT and CAPTURE " + "devices not yet supported.\n"); + printk(KERN_INFO + "v4l2-loopback add() both devices must have the " + "same number (%d != %d).\n", + output_nr, capture_nr); + return -EINVAL; + } + } + + if (idr_find(&v4l2loopback_index_idr, nr)) + return -EEXIST; + + /* initialisation of a new device */ + dprintk("add() creating device #%d\n", nr); + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + /* allocate id, if @id >= 0, we're requesting that specific id */ + if (nr >= 0) { + err = idr_alloc(&v4l2loopback_index_idr, dev, nr, nr + 1, + GFP_KERNEL); + if (err == -ENOSPC) + err = -EEXIST; + } else { + err = idr_alloc(&v4l2loopback_index_idr, dev, 0, 0, GFP_KERNEL); + } + if (err < 0) + goto out_free_dev; + + /* register new device */ + MARK(); + nr = err; + + if (conf && conf->card_label[0]) { + snprintf(dev->card_label, sizeof(dev->card_label), "%s", + conf->card_label); + } else { + snprintf(dev->card_label, sizeof(dev->card_label), + "Dummy video device (0x%04X)", nr); + } + snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name), + "v4l2loopback-%03d", nr); + + err = v4l2_device_register(NULL, &dev->v4l2_dev); + if (err) + goto out_free_idr; + + /* initialise the _video_ device */ + MARK(); + err = -ENOMEM; + dev->vdev = video_device_alloc(); + if (dev->vdev == NULL) + goto out_unregister; + + vdev_priv = kzalloc(sizeof(struct v4l2loopback_private), GFP_KERNEL); + if (vdev_priv == NULL) + goto out_unregister; + + video_set_drvdata(dev->vdev, vdev_priv); + if (video_get_drvdata(dev->vdev) == NULL) + goto out_unregister; + + snprintf(dev->vdev->name, sizeof(dev->vdev->name), "%s", + dev->card_label); + vdev_priv->device_nr = nr; + init_vdev(dev->vdev, nr); + dev->vdev->v4l2_dev = &dev->v4l2_dev; + + /* initialise v4l2-loopback specific parameters */ + MARK(); + dev->announce_all_caps = _announce_all_caps; + dev->min_width = _min_width; + dev->min_height = _min_height; + dev->max_width = _max_width; + dev->max_height = _max_height; + dev->max_openers = _max_openers; + + /* set (initial) pixel and stream format */ + _width = clamp_val(_width, _min_width, _max_width); + _height = clamp_val(_height, _min_height, _max_height); + _fmt = (struct v4l2_format){ + .type = V4L2_BUF_TYPE_VIDEO_CAPTURE, + .fmt.pix = { .width = _width, + .height = _height, + .pixelformat = formats[0].fourcc, + .colorspace = V4L2_COLORSPACE_DEFAULT, + .field = V4L2_FIELD_NONE } + }; + + err = v4l2l_fill_format(&_fmt, _min_width, _max_width, _min_height, + _max_height); + if (err) + /* highly unexpected failure to assign default format */ + goto out_unregister; + dev->pix_format = _fmt.fmt.pix; + init_capture_param(&dev->capture_param); + set_timeperframe(dev, &dev->capture_param.timeperframe); + + /* ctrls parameters */ + dev->keep_format = 0; + dev->sustain_framerate = 0; + dev->timeout_jiffies = 0; + dev->timeout_image_io = 0; + + /* initialise OUTPUT and CAPTURE buffer values */ + dev->image = NULL; + dev->image_size = 0; + dev->buffer_count = _max_buffers; + dev->buffer_size = 0; + dev->used_buffer_count = 0; + INIT_LIST_HEAD(&dev->outbufs_list); + do { + u32 index; + for (index = 0; index < dev->buffer_count; ++index) + INIT_LIST_HEAD(&dev->buffers[index].list_head); + + } while (0); + memset(dev->bufpos2index, 0, sizeof(dev->bufpos2index)); + dev->write_position = 0; + + /* initialise synchronisation data */ + atomic_set(&dev->open_count, 0); + mutex_init(&dev->image_mutex); + spin_lock_init(&dev->lock); + spin_lock_init(&dev->list_lock); + init_waitqueue_head(&dev->read_event); + dev->format_tokens = V4L2L_TOKEN_MASK; + dev->stream_tokens = V4L2L_TOKEN_MASK; + + /* initialise sustain frame rate and timeout parameters, and timers */ + dev->reread_count = 0; + dev->timeout_image = NULL; + dev->timeout_happened = 0; +#ifdef HAVE_TIMER_SETUP + timer_setup(&dev->sustain_timer, sustain_timer_clb, 0); + timer_setup(&dev->timeout_timer, timeout_timer_clb, 0); +#else + setup_timer(&dev->sustain_timer, sustain_timer_clb, nr); + setup_timer(&dev->timeout_timer, timeout_timer_clb, nr); +#endif + + /* initialise the control handler and add controls */ + MARK(); + hdl = &dev->ctrl_handler; + err = v4l2_ctrl_handler_init(hdl, 4); + if (err) + goto out_unregister; + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_keepformat, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_sustainframerate, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeout, NULL); + v4l2_ctrl_new_custom(hdl, &v4l2loopback_ctrl_timeoutimageio, NULL); + if (hdl->error) { + err = hdl->error; + goto out_free_handler; + } + dev->v4l2_dev.ctrl_handler = hdl; + + err = v4l2_ctrl_handler_setup(hdl); + if (err) + goto out_free_handler; + + /* register the device (creates /dev/video*) */ + MARK(); + if (video_register_device(dev->vdev, VFL_TYPE_VIDEO, nr) < 0) { + printk(KERN_ERR + "v4l2-loopback add() failed video_register_device()\n"); + err = -EFAULT; + goto out_free_device; + } + v4l2loopback_create_sysfs(dev->vdev); + /* NOTE: ambivalent if sysfs entries fail */ + + if (ret_nr) + *ret_nr = dev->vdev->num; + return 0; + +out_free_device: + video_device_release(dev->vdev); +out_free_handler: + v4l2_ctrl_handler_free(&dev->ctrl_handler); +out_unregister: + video_set_drvdata(dev->vdev, NULL); + if (vdev_priv != NULL) + kfree(vdev_priv); + v4l2_device_unregister(&dev->v4l2_dev); +out_free_idr: + idr_remove(&v4l2loopback_index_idr, nr); +out_free_dev: + kfree(dev); + return err; +} + +static void v4l2_loopback_remove(struct v4l2_loopback_device *dev) +{ + int device_nr = v4l2loopback_get_vdev_nr(dev->vdev); + mutex_lock(&dev->image_mutex); + free_buffers(dev); + free_timeout_buffer(dev); + mutex_unlock(&dev->image_mutex); + v4l2loopback_remove_sysfs(dev->vdev); + v4l2_ctrl_handler_free(&dev->ctrl_handler); + kfree(video_get_drvdata(dev->vdev)); + video_unregister_device(dev->vdev); + v4l2_device_unregister(&dev->v4l2_dev); + idr_remove(&v4l2loopback_index_idr, device_nr); + kfree(dev); +} + +static long v4l2loopback_control_ioctl(struct file *file, unsigned int cmd, + unsigned long parm) +{ + struct v4l2_loopback_device *dev; + struct v4l2_loopback_config conf; + struct v4l2_loopback_config *confptr = &conf; + int device_nr, capture_nr, output_nr; + int ret; + + ret = mutex_lock_killable(&v4l2loopback_ctl_mutex); + if (ret) + return ret; + + ret = -EINVAL; + switch (cmd) { + default: + ret = -ENOSYS; + break; + /* add a v4l2loopback device (pair), based on the user-provided specs */ + case V4L2LOOPBACK_CTL_ADD: + if (parm) { + if ((ret = copy_from_user(&conf, (void *)parm, + sizeof(conf))) < 0) + break; + } else + confptr = NULL; + ret = v4l2_loopback_add(confptr, &device_nr); + if (ret >= 0) + ret = device_nr; + break; + /* remove a v4l2loopback device (both capture and output) */ + case V4L2LOOPBACK_CTL_REMOVE: + ret = v4l2loopback_lookup((int)parm, &dev); + if (ret >= 0 && dev) { + ret = -EBUSY; + if (dev->open_count.counter > 0) + break; + v4l2_loopback_remove(dev); + ret = 0; + }; + break; + /* get information for a loopback device. + * this is mostly about limits (which cannot be queried directly with VIDIOC_G_FMT and friends + */ + case V4L2LOOPBACK_CTL_QUERY: + if (!parm) + break; + if ((ret = copy_from_user(&conf, (void *)parm, sizeof(conf))) < + 0) + break; + capture_nr = output_nr = conf.output_nr; +#ifdef SPLIT_DEVICES + capture_nr = conf.capture_nr; +#endif + device_nr = (output_nr < 0) ? capture_nr : output_nr; + MARK(); + /* get the device from either capture_nr or output_nr (whatever is valid) */ + if ((ret = v4l2loopback_lookup(device_nr, &dev)) < 0) + break; + MARK(); + /* if we got the device from output_nr and there is a valid capture_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != capture_nr) && (capture_nr >= 0) && + ((ret = v4l2loopback_lookup(capture_nr, 0)) < 0)) + break; + MARK(); + /* if otoh, we got the device from capture_nr and there is a valid output_nr, + * make sure that both refer to the same device (or bail out) + */ + if ((device_nr != output_nr) && (output_nr >= 0) && + ((ret = v4l2loopback_lookup(output_nr, 0)) < 0)) + break; + + /* v4l2_loopback_config identified a single device, so fetch the data */ + snprintf(conf.card_label, sizeof(conf.card_label), "%s", + dev->card_label); + + conf.output_nr = dev->vdev->num; +#ifdef SPLIT_DEVICES + conf.capture_nr = dev->vdev->num; +#endif + conf.min_width = dev->min_width; + conf.min_height = dev->min_height; + conf.max_width = dev->max_width; + conf.max_height = dev->max_height; + conf.announce_all_caps = dev->announce_all_caps; + conf.max_buffers = dev->buffer_count; + conf.max_openers = dev->max_openers; + conf.debug = debug; + MARK(); + if (copy_to_user((void *)parm, &conf, sizeof(conf))) { + ret = -EFAULT; + break; + } + ret = 0; + break; + } + + mutex_unlock(&v4l2loopback_ctl_mutex); + MARK(); + return ret; +} + +/* LINUX KERNEL */ + +static const struct file_operations v4l2loopback_ctl_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = nonseekable_open, + .unlocked_ioctl = v4l2loopback_control_ioctl, + .compat_ioctl = v4l2loopback_control_ioctl, + .llseek = noop_llseek, + // clang-format on +}; + +static struct miscdevice v4l2loopback_misc = { + // clang-format off + .minor = MISC_DYNAMIC_MINOR, + .name = "v4l2loopback", + .fops = &v4l2loopback_ctl_fops, + // clang-format on +}; + +static const struct v4l2_file_operations v4l2_loopback_fops = { + // clang-format off + .owner = THIS_MODULE, + .open = v4l2_loopback_open, + .release = v4l2_loopback_close, + .read = v4l2_loopback_read, + .write = v4l2_loopback_write, + .poll = v4l2_loopback_poll, + .mmap = v4l2_loopback_mmap, + .unlocked_ioctl = video_ioctl2, + // clang-format on +}; + +static const struct v4l2_ioctl_ops v4l2_loopback_ioctl_ops = { + // clang-format off + .vidioc_querycap = &vidioc_querycap, + .vidioc_enum_framesizes = &vidioc_enum_framesizes, + .vidioc_enum_frameintervals = &vidioc_enum_frameintervals, + + .vidioc_enum_output = &vidioc_enum_output, + .vidioc_g_output = &vidioc_g_output, + .vidioc_s_output = &vidioc_s_output, + + .vidioc_enum_input = &vidioc_enum_input, + .vidioc_g_input = &vidioc_g_input, + .vidioc_s_input = &vidioc_s_input, + + .vidioc_enum_fmt_vid_cap = &vidioc_enum_fmt_cap, + .vidioc_g_fmt_vid_cap = &vidioc_g_fmt_cap, + .vidioc_s_fmt_vid_cap = &vidioc_s_fmt_cap, + .vidioc_try_fmt_vid_cap = &vidioc_try_fmt_cap, + + .vidioc_enum_fmt_vid_out = &vidioc_enum_fmt_out, + .vidioc_s_fmt_vid_out = &vidioc_s_fmt_out, + .vidioc_g_fmt_vid_out = &vidioc_g_fmt_out, + .vidioc_try_fmt_vid_out = &vidioc_try_fmt_out, + +#ifdef V4L2L_OVERLAY + .vidioc_s_fmt_vid_overlay = &vidioc_s_fmt_overlay, + .vidioc_g_fmt_vid_overlay = &vidioc_g_fmt_overlay, +#endif + +#ifdef V4L2LOOPBACK_WITH_STD + .vidioc_s_std = &vidioc_s_std, + .vidioc_g_std = &vidioc_g_std, + .vidioc_querystd = &vidioc_querystd, +#endif /* V4L2LOOPBACK_WITH_STD */ + + .vidioc_g_parm = &vidioc_g_parm, + .vidioc_s_parm = &vidioc_s_parm, + + .vidioc_reqbufs = &vidioc_reqbufs, + .vidioc_querybuf = &vidioc_querybuf, + .vidioc_qbuf = &vidioc_qbuf, + .vidioc_dqbuf = &vidioc_dqbuf, + + .vidioc_streamon = &vidioc_streamon, + .vidioc_streamoff = &vidioc_streamoff, + +#ifdef CONFIG_VIDEO_V4L1_COMPAT + .vidiocgmbuf = &vidiocgmbuf, +#endif + + .vidioc_subscribe_event = &vidioc_subscribe_event, + .vidioc_unsubscribe_event = &v4l2_event_unsubscribe, + // clang-format on +}; + +static int free_device_cb(int id, void *ptr, void *data) +{ + struct v4l2_loopback_device *dev = ptr; + v4l2_loopback_remove(dev); + return 0; +} +static void free_devices(void) +{ + idr_for_each(&v4l2loopback_index_idr, &free_device_cb, NULL); + idr_destroy(&v4l2loopback_index_idr); +} + +static int __init v4l2loopback_init_module(void) +{ + const u32 min_width = V4L2LOOPBACK_SIZE_MIN_WIDTH; + const u32 min_height = V4L2LOOPBACK_SIZE_MIN_HEIGHT; + int err; + int i; + MARK(); + + err = misc_register(&v4l2loopback_misc); + if (err < 0) + return err; + + if (devices < 0) { + devices = 1; + + /* try guessing the devices from the "video_nr" parameter */ + for (i = MAX_DEVICES - 1; i >= 0; i--) { + if (video_nr[i] >= 0) { + devices = i + 1; + break; + } + } + } + + if (devices > MAX_DEVICES) { + devices = MAX_DEVICES; + printk(KERN_INFO + "v4l2-loopback init() number of initial devices is " + "limited to: %d\n", + MAX_DEVICES); + } + + if (max_buffers > MAX_BUFFERS) { + max_buffers = MAX_BUFFERS; + printk(KERN_INFO + "v4l2-loopback init() number of buffers is limited " + "to: %d\n", + MAX_BUFFERS); + } + + if (max_openers < 0) { + printk(KERN_INFO + "v4l2-loopback init() allowing %d openers rather " + "than %d\n", + 2, max_openers); + max_openers = 2; + } + + if (max_width < min_width) { + max_width = V4L2LOOPBACK_SIZE_DEFAULT_MAX_WIDTH; + printk(KERN_INFO "v4l2-loopback init() using max_width %d\n", + max_width); + } + if (max_height < min_height) { + max_height = V4L2LOOPBACK_SIZE_DEFAULT_MAX_HEIGHT; + printk(KERN_INFO "v4l2-loopback init() using max_height %d\n", + max_height); + } + + for (i = 0; i < devices; i++) { + struct v4l2_loopback_config cfg = { + // clang-format off + .output_nr = video_nr[i], +#ifdef SPLIT_DEVICES + .capture_nr = video_nr[i], +#endif + .min_width = min_width, + .min_height = min_height, + .max_width = max_width, + .max_height = max_height, + .announce_all_caps = (!exclusive_caps[i]), + .max_buffers = max_buffers, + .max_openers = max_openers, + .debug = debug, + // clang-format on + }; + cfg.card_label[0] = 0; + if (card_label[i]) + snprintf(cfg.card_label, sizeof(cfg.card_label), "%s", + card_label[i]); + err = v4l2_loopback_add(&cfg, 0); + if (err) { + free_devices(); + goto error; + } + } + + dprintk("module installed\n"); + + printk(KERN_INFO "v4l2-loopback driver version %d.%d.%d%s loaded\n", + // clang-format off + (V4L2LOOPBACK_VERSION_CODE >> 16) & 0xff, + (V4L2LOOPBACK_VERSION_CODE >> 8) & 0xff, + (V4L2LOOPBACK_VERSION_CODE ) & 0xff, +#ifdef SNAPSHOT_VERSION + " (" __stringify(SNAPSHOT_VERSION) ")" +#else + "" +#endif + ); + // clang-format on + + return 0; +error: + misc_deregister(&v4l2loopback_misc); + return err; +} + +static void v4l2loopback_cleanup_module(void) +{ + MARK(); + /* unregister the device -> it deletes /dev/video* */ + free_devices(); + /* and get rid of /dev/v4l2loopback */ + misc_deregister(&v4l2loopback_misc); + dprintk("module removed\n"); +} + +MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR); + +module_init(v4l2loopback_init_module); +module_exit(v4l2loopback_cleanup_module); diff --git a/drivers/media/v4l2-core/v4l2loopback.h b/drivers/media/v4l2-core/v4l2loopback.h new file mode 100644 index 000000000000..ec0be6cbe97d --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * v4l2loopback.h + * + * Written by IOhannes m zmölnig, 7/1/20. + * + * Copyright 2020 by IOhannes m zmölnig. Redistribution of this file is + * permitted under the GNU General Public License. + */ +#ifndef _V4L2LOOPBACK_H +#define _V4L2LOOPBACK_H + +#define V4L2LOOPBACK_VERSION_MAJOR 0 +#define V4L2LOOPBACK_VERSION_MINOR 14 +#define V4L2LOOPBACK_VERSION_BUGFIX 0 + +/* /dev/v4l2loopback interface */ + +struct v4l2_loopback_config { + /** + * the device-number (/dev/video) + * V4L2LOOPBACK_CTL_ADD: + * setting this to a value<0, will allocate an available one + * if nr>=0 and the device already exists, the ioctl will EEXIST + * if output_nr and capture_nr are the same, only a single device will be created + * NOTE: currently split-devices (where output_nr and capture_nr differ) + * are not implemented yet. + * until then, requesting different device-IDs will result in EINVAL. + * + * V4L2LOOPBACK_CTL_QUERY: + * either both output_nr and capture_nr must refer to the same loopback, + * or one (and only one) of them must be -1 + * + */ + int output_nr; + int unused; /*capture_nr;*/ + + /** + * a nice name for your device + * if (*card_label)==0, an automatic name is assigned + */ + char card_label[32]; + + /** + * allowed frame size + * if too low, default values are used + */ + unsigned int min_width; + unsigned int max_width; + unsigned int min_height; + unsigned int max_height; + + /** + * number of buffers to allocate for the queue + * if set to <=0, default values are used + */ + int max_buffers; + + /** + * how many consumers are allowed to open this device concurrently + * if set to <=0, default values are used + */ + int max_openers; + + /** + * set the debugging level for this device + */ + int debug; + + /** + * whether to announce OUTPUT/CAPTURE capabilities exclusively + * for this device or not + * (!exclusive_caps) + * NOTE: this is going to be removed once separate output/capture + * devices are implemented + */ + int announce_all_caps; +}; + +/* a pointer to a (struct v4l2_loopback_config) that has all values you wish to impose on the + * to-be-created device set. + * if the ptr is NULL, a new device is created with default values at the driver's discretion. + * + * returns the device_nr of the OUTPUT device (which can be used with V4L2LOOPBACK_CTL_QUERY, + * to get more information on the device) + */ +#define V4L2LOOPBACK_CTL_ADD 0x4C80 + +/* a pointer to a (struct v4l2_loopback_config) that has output_nr and/or capture_nr set + * (the two values must either refer to video-devices associated with the same loopback device + * or exactly one of them must be <0 + */ +#define V4L2LOOPBACK_CTL_QUERY 0x4C82 + +/* the device-number (either CAPTURE or OUTPUT) associated with the loopback-device */ +#define V4L2LOOPBACK_CTL_REMOVE 0x4C81 + +#endif /* _V4L2LOOPBACK_H */ diff --git a/drivers/media/v4l2-core/v4l2loopback_formats.h b/drivers/media/v4l2-core/v4l2loopback_formats.h new file mode 100644 index 000000000000..d855a3796554 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2loopback_formats.h @@ -0,0 +1,445 @@ +static const struct v4l2l_format formats[] = { +#ifndef V4L2_PIX_FMT_VP9 +#define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0') +#endif +#ifndef V4L2_PIX_FMT_HEVC +#define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C') +#endif + + /* here come the packed formats */ + { + .name = "32 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR32, + .depth = 32, + .flags = 0, + }, + { + .name = "32 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB32, + .depth = 32, + .flags = 0, + }, + { + .name = "24 bpp RGB, le", + .fourcc = V4L2_PIX_FMT_BGR24, + .depth = 24, + .flags = 0, + }, + { + .name = "24 bpp RGB, be", + .fourcc = V4L2_PIX_FMT_RGB24, + .depth = 24, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_ABGR32 + { + .name = "32 bpp RGBA, le", + .fourcc = V4L2_PIX_FMT_ABGR32, + .depth = 32, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_RGBA32 + { + .name = "32 bpp RGBA", + .fourcc = V4L2_PIX_FMT_RGBA32, + .depth = 32, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_RGB332 + { + .name = "8 bpp RGB-3-3-2", + .fourcc = V4L2_PIX_FMT_RGB332, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB332 */ +#ifdef V4L2_PIX_FMT_RGB444 + { + .name = "16 bpp RGB (xxxxrrrr ggggbbbb)", + .fourcc = V4L2_PIX_FMT_RGB444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB444 */ +#ifdef V4L2_PIX_FMT_RGB555 + { + .name = "16 bpp RGB-5-5-5", + .fourcc = V4L2_PIX_FMT_RGB555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555 */ +#ifdef V4L2_PIX_FMT_RGB565 + { + .name = "16 bpp RGB-5-6-5", + .fourcc = V4L2_PIX_FMT_RGB565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565 */ +#ifdef V4L2_PIX_FMT_RGB555X + { + .name = "16 bpp RGB-5-5-5 BE", + .fourcc = V4L2_PIX_FMT_RGB555X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB555X */ +#ifdef V4L2_PIX_FMT_RGB565X + { + .name = "16 bpp RGB-5-6-5 BE", + .fourcc = V4L2_PIX_FMT_RGB565X, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_RGB565X */ +#ifdef V4L2_PIX_FMT_BGR666 + { + .name = "18 bpp BGR-6-6-6", + .fourcc = V4L2_PIX_FMT_BGR666, + .depth = 18, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_BGR666 */ + { + .name = "4:2:2, packed, YUYV", + .fourcc = V4L2_PIX_FMT_YUYV, + .depth = 16, + .flags = 0, + }, + { + .name = "4:2:2, packed, UYVY", + .fourcc = V4L2_PIX_FMT_UYVY, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YVYU + { + .name = "4:2:2, packed YVYU", + .fourcc = V4L2_PIX_FMT_YVYU, + .depth = 16, + .flags = 0, + }, +#endif +#ifdef V4L2_PIX_FMT_VYUY + { + .name = "4:2:2, packed VYUY", + .fourcc = V4L2_PIX_FMT_VYUY, + .depth = 16, + .flags = 0, + }, +#endif + { + .name = "4:2:2, packed YYUV", + .fourcc = V4L2_PIX_FMT_YYUV, + .depth = 16, + .flags = 0, + }, + { + .name = "YUV-8-8-8-8", + .fourcc = V4L2_PIX_FMT_YUV32, + .depth = 32, + .flags = 0, + }, + { + .name = "8 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_GREY, + .depth = 8, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_Y4 + { + .name = "4 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y4, + .depth = 4, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y4 */ +#ifdef V4L2_PIX_FMT_Y6 + { + .name = "6 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y6, + .depth = 6, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y6 */ +#ifdef V4L2_PIX_FMT_Y10 + { + .name = "10 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y10, + .depth = 10, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y10 */ +#ifdef V4L2_PIX_FMT_Y12 + { + .name = "12 bpp Greyscale", + .fourcc = V4L2_PIX_FMT_Y12, + .depth = 12, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_Y12 */ + { + .name = "16 bpp, Greyscale", + .fourcc = V4L2_PIX_FMT_Y16, + .depth = 16, + .flags = 0, + }, +#ifdef V4L2_PIX_FMT_YUV444 + { + .name = "16 bpp xxxxyyyy uuuuvvvv", + .fourcc = V4L2_PIX_FMT_YUV444, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV444 */ +#ifdef V4L2_PIX_FMT_YUV555 + { + .name = "16 bpp YUV-5-5-5", + .fourcc = V4L2_PIX_FMT_YUV555, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV555 */ +#ifdef V4L2_PIX_FMT_YUV565 + { + .name = "16 bpp YUV-5-6-5", + .fourcc = V4L2_PIX_FMT_YUV565, + .depth = 16, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_YUV565 */ + +/* bayer formats */ +#ifdef V4L2_PIX_FMT_SRGGB8 + { + .name = "Bayer RGGB 8bit", + .fourcc = V4L2_PIX_FMT_SRGGB8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SRGGB8 */ +#ifdef V4L2_PIX_FMT_SGRBG8 + { + .name = "Bayer GRBG 8bit", + .fourcc = V4L2_PIX_FMT_SGRBG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGRBG8 */ +#ifdef V4L2_PIX_FMT_SGBRG8 + { + .name = "Bayer GBRG 8bit", + .fourcc = V4L2_PIX_FMT_SGBRG8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SGBRG8 */ +#ifdef V4L2_PIX_FMT_SBGGR8 + { + .name = "Bayer BA81 8bit", + .fourcc = V4L2_PIX_FMT_SBGGR8, + .depth = 8, + .flags = 0, + }, +#endif /* V4L2_PIX_FMT_SBGGR8 */ + + /* here come the planar formats */ + { + .name = "4:1:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cr-Cb", + .fourcc = V4L2_PIX_FMT_YVU420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:1:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV410, + .depth = 9, + .flags = FORMAT_FLAGS_PLANAR, + }, + { + .name = "4:2:0, planar, Y-Cb-Cr", + .fourcc = V4L2_PIX_FMT_YUV420, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#ifdef V4L2_PIX_FMT_YUV422P + { + .name = "16 bpp YVU422 planar", + .fourcc = V4L2_PIX_FMT_YUV422P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV422P */ +#ifdef V4L2_PIX_FMT_YUV411P + { + .name = "16 bpp YVU411 planar", + .fourcc = V4L2_PIX_FMT_YUV411P, + .depth = 16, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_YUV411P */ +#ifdef V4L2_PIX_FMT_Y41P + { + .name = "12 bpp YUV 4:1:1", + .fourcc = V4L2_PIX_FMT_Y41P, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_Y41P */ +#ifdef V4L2_PIX_FMT_NV12 + { + .name = "12 bpp Y/CbCr 4:2:0 ", + .fourcc = V4L2_PIX_FMT_NV12, + .depth = 12, + .flags = FORMAT_FLAGS_PLANAR, + }, +#endif /* V4L2_PIX_FMT_NV12 */ + +/* here come the compressed formats */ + +#ifdef V4L2_PIX_FMT_MJPEG + { + .name = "Motion-JPEG", + .fourcc = V4L2_PIX_FMT_MJPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MJPEG */ +#ifdef V4L2_PIX_FMT_JPEG + { + .name = "JFIF JPEG", + .fourcc = V4L2_PIX_FMT_JPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_JPEG */ +#ifdef V4L2_PIX_FMT_DV + { + .name = "DV1394", + .fourcc = V4L2_PIX_FMT_DV, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_DV */ +#ifdef V4L2_PIX_FMT_MPEG + { + .name = "MPEG-1/2/4 Multiplexed", + .fourcc = V4L2_PIX_FMT_MPEG, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG */ +#ifdef V4L2_PIX_FMT_H264 + { + .name = "H264 with start codes", + .fourcc = V4L2_PIX_FMT_H264, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264 */ +#ifdef V4L2_PIX_FMT_H264_NO_SC + { + .name = "H264 without start codes", + .fourcc = V4L2_PIX_FMT_H264_NO_SC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_NO_SC */ +#ifdef V4L2_PIX_FMT_H264_MVC + { + .name = "H264 MVC", + .fourcc = V4L2_PIX_FMT_H264_MVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H264_MVC */ +#ifdef V4L2_PIX_FMT_H263 + { + .name = "H263", + .fourcc = V4L2_PIX_FMT_H263, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_H263 */ +#ifdef V4L2_PIX_FMT_MPEG1 + { + .name = "MPEG-1 ES", + .fourcc = V4L2_PIX_FMT_MPEG1, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG1 */ +#ifdef V4L2_PIX_FMT_MPEG2 + { + .name = "MPEG-2 ES", + .fourcc = V4L2_PIX_FMT_MPEG2, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG2 */ +#ifdef V4L2_PIX_FMT_MPEG4 + { + .name = "MPEG-4 part 2 ES", + .fourcc = V4L2_PIX_FMT_MPEG4, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_MPEG4 */ +#ifdef V4L2_PIX_FMT_XVID + { + .name = "Xvid", + .fourcc = V4L2_PIX_FMT_XVID, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_XVID */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_G + { + .name = "SMPTE 421M Annex G compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_G, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_G */ +#ifdef V4L2_PIX_FMT_VC1_ANNEX_L + { + .name = "SMPTE 421M Annex L compliant stream", + .fourcc = V4L2_PIX_FMT_VC1_ANNEX_L, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VC1_ANNEX_L */ +#ifdef V4L2_PIX_FMT_VP8 + { + .name = "VP8", + .fourcc = V4L2_PIX_FMT_VP8, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP8 */ +#ifdef V4L2_PIX_FMT_VP9 + { + .name = "VP9", + .fourcc = V4L2_PIX_FMT_VP9, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_VP9 */ +#ifdef V4L2_PIX_FMT_HEVC + { + .name = "HEVC", + .fourcc = V4L2_PIX_FMT_HEVC, + .depth = 32, + .flags = FORMAT_FLAGS_COMPRESSED, + }, +#endif /* V4L2_PIX_FMT_HEVC */ +}; diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 038ccbd9e3ba..de5e4f5145af 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -1,4 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 +ifdef CONFIG_X86_64 +ifdef CONFIG_SATA_AHCI +obj-y += intel-nvme-remap.o +endif +endif + obj-$(CONFIG_PCIE_CADENCE) += cadence/ obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o obj-$(CONFIG_PCI_IXP4XX) += pci-ixp4xx.o diff --git a/drivers/pci/controller/intel-nvme-remap.c b/drivers/pci/controller/intel-nvme-remap.c new file mode 100644 index 000000000000..e105e6f5cc91 --- /dev/null +++ b/drivers/pci/controller/intel-nvme-remap.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Intel remapped NVMe device support. + * + * Copyright (c) 2019 Endless Mobile, Inc. + * Author: Daniel Drake + * + * Some products ship by default with the SATA controller in "RAID" or + * "Intel RST Premium With Intel Optane System Acceleration" mode. Under this + * mode, which we refer to as "remapped NVMe" mode, any installed NVMe + * devices disappear from the PCI bus, and instead their I/O memory becomes + * available within the AHCI device BARs. + * + * This scheme is understood to be a way of avoiding usage of the standard + * Windows NVMe driver under that OS, instead mandating usage of Intel's + * driver instead, which has better power management, and presumably offers + * some RAID/disk-caching solutions too. + * + * Here in this driver, we support the remapped NVMe mode by claiming the + * AHCI device and creating a fake PCIe root port. On the new bus, the + * original AHCI device is exposed with only minor tweaks. Then, fake PCI + * devices corresponding to the remapped NVMe devices are created. The usual + * ahci and nvme drivers are then expected to bind to these devices and + * operate as normal. + * + * The PCI configuration space for the NVMe devices is completely + * unavailable, so we fake a minimal one and hope for the best. + * + * Interrupts are shared between the AHCI and NVMe devices. For simplicity, + * we only support the legacy interrupt here, although MSI support + * could potentially be added later. + */ + +#define MODULE_NAME "intel-nvme-remap" + +#include +#include +#include +#include +#include + +#define AHCI_PCI_BAR_STANDARD 5 + +struct nvme_remap_dev { + struct pci_dev *dev; /* AHCI device */ + struct pci_bus *bus; /* our fake PCI bus */ + struct pci_sysdata sysdata; + int irq_base; /* our fake interrupts */ + + /* + * When we detect an all-ones write to a BAR register, this flag + * is set, so that we return the BAR size on the next read (a + * standard PCI behaviour). + * This includes the assumption that an all-ones BAR write is + * immediately followed by a read of the same register. + */ + bool bar_sizing; + + /* + * Resources copied from the AHCI device, to be regarded as + * resources on our fake bus. + */ + struct resource ahci_resources[PCI_NUM_RESOURCES]; + + /* Resources corresponding to the NVMe devices. */ + struct resource remapped_dev_mem[AHCI_MAX_REMAP]; + + /* Number of remapped NVMe devices found. */ + int num_remapped_devices; +}; + +static inline struct nvme_remap_dev *nrdev_from_bus(struct pci_bus *bus) +{ + return container_of(bus->sysdata, struct nvme_remap_dev, sysdata); +} + + +/******** PCI configuration space **********/ + +/* + * Helper macros for tweaking returned contents of PCI configuration space. + * + * value contains len bytes of data read from reg. + * If fixup_reg is included in that range, fix up the contents of that + * register to fixed_value. + */ +#define NR_FIX8(fixup_reg, fixed_value) do { \ + if (reg <= fixup_reg && fixup_reg < reg + len) \ + ((u8 *) value)[fixup_reg - reg] = (u8) (fixed_value); \ + } while (0) + +#define NR_FIX16(fixup_reg, fixed_value) do { \ + NR_FIX8(fixup_reg, fixed_value); \ + NR_FIX8(fixup_reg + 1, fixed_value >> 8); \ + } while (0) + +#define NR_FIX24(fixup_reg, fixed_value) do { \ + NR_FIX8(fixup_reg, fixed_value); \ + NR_FIX8(fixup_reg + 1, fixed_value >> 8); \ + NR_FIX8(fixup_reg + 2, fixed_value >> 16); \ + } while (0) + +#define NR_FIX32(fixup_reg, fixed_value) do { \ + NR_FIX16(fixup_reg, (u16) fixed_value); \ + NR_FIX16(fixup_reg + 2, fixed_value >> 16); \ + } while (0) + +/* + * Read PCI config space of the slot 0 (AHCI) device. + * We pass through the read request to the underlying device, but + * tweak the results in some cases. + */ +static int nvme_remap_pci_read_slot0(struct pci_bus *bus, int reg, + int len, u32 *value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct pci_bus *ahci_dev_bus = nrdev->dev->bus; + int ret; + + ret = ahci_dev_bus->ops->read(ahci_dev_bus, nrdev->dev->devfn, + reg, len, value); + if (ret) + return ret; + + /* + * Adjust the device class, to prevent this driver from attempting to + * additionally probe the device we're simulating here. + */ + NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_SATA_AHCI); + + /* + * Unset interrupt pin, otherwise ACPI tries to find routing + * info for our virtual IRQ, fails, and complains. + */ + NR_FIX8(PCI_INTERRUPT_PIN, 0); + + /* + * Truncate the AHCI BAR to not include the region that covers the + * hidden devices. This will cause the ahci driver to successfully + * probe th new device (instead of handing it over to this driver). + */ + if (nrdev->bar_sizing) { + NR_FIX32(PCI_BASE_ADDRESS_5, ~(SZ_16K - 1)); + nrdev->bar_sizing = false; + } + + return PCIBIOS_SUCCESSFUL; +} + +/* + * Read PCI config space of a remapped device. + * Since the original PCI config space is inaccessible, we provide a minimal, + * fake config space instead. + */ +static int nvme_remap_pci_read_remapped(struct pci_bus *bus, unsigned int port, + int reg, int len, u32 *value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct resource *remapped_mem; + + if (port > nrdev->num_remapped_devices) + return PCIBIOS_DEVICE_NOT_FOUND; + + *value = 0; + remapped_mem = &nrdev->remapped_dev_mem[port - 1]; + + /* Set a Vendor ID, otherwise Linux assumes no device is present */ + NR_FIX16(PCI_VENDOR_ID, PCI_VENDOR_ID_INTEL); + + /* Always appear on & bus mastering */ + NR_FIX16(PCI_COMMAND, PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + + /* Set class so that nvme driver probes us */ + NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_EXPRESS); + + if (nrdev->bar_sizing) { + NR_FIX32(PCI_BASE_ADDRESS_0, + ~(resource_size(remapped_mem) - 1)); + nrdev->bar_sizing = false; + } else { + resource_size_t mem_start = remapped_mem->start; + + mem_start |= PCI_BASE_ADDRESS_MEM_TYPE_64; + NR_FIX32(PCI_BASE_ADDRESS_0, mem_start); + mem_start >>= 32; + NR_FIX32(PCI_BASE_ADDRESS_1, mem_start); + } + + return PCIBIOS_SUCCESSFUL; +} + +/* Read PCI configuration space. */ +static int nvme_remap_pci_read(struct pci_bus *bus, unsigned int devfn, + int reg, int len, u32 *value) +{ + if (PCI_SLOT(devfn) == 0) + return nvme_remap_pci_read_slot0(bus, reg, len, value); + else + return nvme_remap_pci_read_remapped(bus, PCI_SLOT(devfn), + reg, len, value); +} + +/* + * Write PCI config space of the slot 0 (AHCI) device. + * Apart from the special case of BAR sizing, we disable all writes. + * Otherwise, the ahci driver could make changes (e.g. unset PCI bus master) + * that would affect the operation of the NVMe devices. + */ +static int nvme_remap_pci_write_slot0(struct pci_bus *bus, int reg, + int len, u32 value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + struct pci_bus *ahci_dev_bus = nrdev->dev->bus; + + if (reg >= PCI_BASE_ADDRESS_0 && reg <= PCI_BASE_ADDRESS_5) { + /* + * Writing all-ones to a BAR means that the size of the + * memory region is being checked. Flag this so that we can + * reply with an appropriate size on the next read. + */ + if (value == ~0) + nrdev->bar_sizing = true; + + return ahci_dev_bus->ops->write(ahci_dev_bus, + nrdev->dev->devfn, + reg, len, value); + } + + return PCIBIOS_SET_FAILED; +} + +/* + * Write PCI config space of a remapped device. + * Since the original PCI config space is inaccessible, we reject all + * writes, except for the special case of BAR probing. + */ +static int nvme_remap_pci_write_remapped(struct pci_bus *bus, + unsigned int port, + int reg, int len, u32 value) +{ + struct nvme_remap_dev *nrdev = nrdev_from_bus(bus); + + if (port > nrdev->num_remapped_devices) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* + * Writing all-ones to a BAR means that the size of the memory + * region is being checked. Flag this so that we can reply with + * an appropriate size on the next read. + */ + if (value == ~0 && reg >= PCI_BASE_ADDRESS_0 + && reg <= PCI_BASE_ADDRESS_5) { + nrdev->bar_sizing = true; + return PCIBIOS_SUCCESSFUL; + } + + return PCIBIOS_SET_FAILED; +} + +/* Write PCI configuration space. */ +static int nvme_remap_pci_write(struct pci_bus *bus, unsigned int devfn, + int reg, int len, u32 value) +{ + if (PCI_SLOT(devfn) == 0) + return nvme_remap_pci_write_slot0(bus, reg, len, value); + else + return nvme_remap_pci_write_remapped(bus, PCI_SLOT(devfn), + reg, len, value); +} + +static struct pci_ops nvme_remap_pci_ops = { + .read = nvme_remap_pci_read, + .write = nvme_remap_pci_write, +}; + + +/******** Initialization & exit **********/ + +/* + * Find a PCI domain ID to use for our fake bus. + * Start at 0x10000 to not clash with ACPI _SEG domains (16 bits). + */ +static int find_free_domain(void) +{ + int domain = 0xffff; + struct pci_bus *bus = NULL; + + while ((bus = pci_find_next_bus(bus)) != NULL) + domain = max_t(int, domain, pci_domain_nr(bus)); + + return domain + 1; +} + +static int find_remapped_devices(struct nvme_remap_dev *nrdev, + struct list_head *resources) +{ + void __iomem *mmio; + int i, count = 0; + u32 cap; + + mmio = pcim_iomap(nrdev->dev, AHCI_PCI_BAR_STANDARD, + pci_resource_len(nrdev->dev, + AHCI_PCI_BAR_STANDARD)); + if (!mmio) + return -ENODEV; + + /* Check if this device might have remapped nvme devices. */ + if (pci_resource_len(nrdev->dev, AHCI_PCI_BAR_STANDARD) < SZ_512K || + !(readl(mmio + AHCI_VSCAP) & 1)) + return -ENODEV; + + cap = readq(mmio + AHCI_REMAP_CAP); + for (i = AHCI_MAX_REMAP-1; i >= 0; i--) { + struct resource *remapped_mem; + + if ((cap & (1 << i)) == 0) + continue; + if (readl(mmio + ahci_remap_dcc(i)) + != PCI_CLASS_STORAGE_EXPRESS) + continue; + + /* We've found a remapped device */ + remapped_mem = &nrdev->remapped_dev_mem[count++]; + remapped_mem->start = + pci_resource_start(nrdev->dev, AHCI_PCI_BAR_STANDARD) + + ahci_remap_base(i); + remapped_mem->end = remapped_mem->start + + AHCI_REMAP_N_SIZE - 1; + remapped_mem->flags = IORESOURCE_MEM | IORESOURCE_PCI_FIXED; + pci_add_resource(resources, remapped_mem); + } + + pcim_iounmap(nrdev->dev, mmio); + + if (count == 0) + return -ENODEV; + + nrdev->num_remapped_devices = count; + dev_info(&nrdev->dev->dev, "Found %d remapped NVMe devices\n", + nrdev->num_remapped_devices); + return 0; +} + +static void nvme_remap_remove_root_bus(void *data) +{ + struct pci_bus *bus = data; + + pci_stop_root_bus(bus); + pci_remove_root_bus(bus); +} + +static int nvme_remap_probe(struct pci_dev *dev, + const struct pci_device_id *id) +{ + struct nvme_remap_dev *nrdev; + LIST_HEAD(resources); + int i; + int ret; + struct pci_dev *child; + + nrdev = devm_kzalloc(&dev->dev, sizeof(*nrdev), GFP_KERNEL); + nrdev->sysdata.domain = find_free_domain(); + nrdev->sysdata.nvme_remap_dev = dev; + nrdev->dev = dev; + pci_set_drvdata(dev, nrdev); + + ret = pcim_enable_device(dev); + if (ret < 0) + return ret; + + pci_set_master(dev); + + ret = find_remapped_devices(nrdev, &resources); + if (ret) + return ret; + + /* Add resources from the original AHCI device */ + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = &dev->resource[i]; + + if (res->start) { + struct resource *nr_res = &nrdev->ahci_resources[i]; + + nr_res->start = res->start; + nr_res->end = res->end; + nr_res->flags = res->flags; + pci_add_resource(&resources, nr_res); + } + } + + /* Create virtual interrupts */ + nrdev->irq_base = devm_irq_alloc_descs(&dev->dev, -1, 0, + nrdev->num_remapped_devices + 1, + 0); + if (nrdev->irq_base < 0) + return nrdev->irq_base; + + /* Create and populate PCI bus */ + nrdev->bus = pci_create_root_bus(&dev->dev, 0, &nvme_remap_pci_ops, + &nrdev->sysdata, &resources); + if (!nrdev->bus) + return -ENODEV; + + if (devm_add_action_or_reset(&dev->dev, nvme_remap_remove_root_bus, + nrdev->bus)) + return -ENOMEM; + + /* We don't support sharing MSI interrupts between these devices */ + nrdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI; + + pci_scan_child_bus(nrdev->bus); + + list_for_each_entry(child, &nrdev->bus->devices, bus_list) { + /* + * Prevent PCI core from trying to move memory BARs around. + * The hidden NVMe devices are at fixed locations. + */ + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + struct resource *res = &child->resource[i]; + + if (res->flags & IORESOURCE_MEM) + res->flags |= IORESOURCE_PCI_FIXED; + } + + /* Share the legacy IRQ between all devices */ + child->irq = dev->irq; + } + + pci_assign_unassigned_bus_resources(nrdev->bus); + pci_bus_add_devices(nrdev->bus); + + return 0; +} + +static const struct pci_device_id nvme_remap_ids[] = { + /* + * Match all Intel RAID controllers. + * + * There's overlap here with the set of devices detected by the ahci + * driver, but ahci will only successfully probe when there + * *aren't* any remapped NVMe devices, and this driver will only + * successfully probe when there *are* remapped NVMe devices that + * need handling. + */ + { + PCI_VDEVICE(INTEL, PCI_ANY_ID), + .class = PCI_CLASS_STORAGE_RAID << 8, + .class_mask = 0xffffff00, + }, + {0,} +}; +MODULE_DEVICE_TABLE(pci, nvme_remap_ids); + +static struct pci_driver nvme_remap_drv = { + .name = MODULE_NAME, + .id_table = nvme_remap_ids, + .probe = nvme_remap_probe, +}; +module_pci_driver(nvme_remap_drv); + +MODULE_AUTHOR("Daniel Drake "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 94daca15a096..4822a23dd357 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3747,6 +3747,106 @@ static void quirk_no_bus_reset(struct pci_dev *dev) dev->dev_flags |= PCI_DEV_FLAGS_NO_BUS_RESET; } +static bool acs_on_downstream; +static bool acs_on_multifunction; + +#define NUM_ACS_IDS 16 +struct acs_on_id { + unsigned short vendor; + unsigned short device; +}; +static struct acs_on_id acs_on_ids[NUM_ACS_IDS]; +static u8 max_acs_id; + +static __init int pcie_acs_override_setup(char *p) +{ + if (!p) + return -EINVAL; + + while (*p) { + if (!strncmp(p, "downstream", 10)) + acs_on_downstream = true; + if (!strncmp(p, "multifunction", 13)) + acs_on_multifunction = true; + if (!strncmp(p, "id:", 3)) { + char opt[5]; + int ret; + long val; + + if (max_acs_id >= NUM_ACS_IDS - 1) { + pr_warn("Out of PCIe ACS override slots (%d)\n", + NUM_ACS_IDS); + goto next; + } + + p += 3; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].vendor = val; + + p += strcspn(p, ":"); + if (*p != ':') { + pr_warn("PCIe ACS invalid ID\n"); + goto next; + } + + p++; + snprintf(opt, 5, "%s", p); + ret = kstrtol(opt, 16, &val); + if (ret) { + pr_warn("PCIe ACS ID parse error %d\n", ret); + goto next; + } + acs_on_ids[max_acs_id].device = val; + max_acs_id++; + } +next: + p += strcspn(p, ","); + if (*p == ',') + p++; + } + + if (acs_on_downstream || acs_on_multifunction || max_acs_id) + pr_warn("Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA\n"); + + return 0; +} +early_param("pcie_acs_override", pcie_acs_override_setup); + +static int pcie_acs_overrides(struct pci_dev *dev, u16 acs_flags) +{ + int i; + + /* Never override ACS for legacy devices or devices with ACS caps */ + if (!pci_is_pcie(dev) || + pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS)) + return -ENOTTY; + + for (i = 0; i < max_acs_id; i++) + if (acs_on_ids[i].vendor == dev->vendor && + acs_on_ids[i].device == dev->device) + return 1; + + switch (pci_pcie_type(dev)) { + case PCI_EXP_TYPE_DOWNSTREAM: + case PCI_EXP_TYPE_ROOT_PORT: + if (acs_on_downstream) + return 1; + break; + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_UPSTREAM: + case PCI_EXP_TYPE_LEG_END: + case PCI_EXP_TYPE_RC_END: + if (acs_on_multifunction && dev->multifunction) + return 1; + } + + return -ENOTTY; +} /* * Some NVIDIA GPU devices do not work with bus reset, SBR needs to be * prevented for those affected devices. @@ -5171,6 +5271,7 @@ static const struct pci_dev_acs_enabled { { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs }, /* Wangxun nics */ { PCI_VENDOR_ID_WANGXUN, PCI_ANY_ID, pci_quirk_wangxun_nic_acs }, + { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides }, { 0 } }; diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 5a3c670aec27..831ffcf7c730 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -1521,4 +1521,6 @@ endif # SCSI_LOWLEVEL source "drivers/scsi/device_handler/Kconfig" +source "drivers/scsi/vhba/Kconfig" + endmenu diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index 16de3e41f94c..4e88f6e3e67b 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -152,6 +152,7 @@ obj-$(CONFIG_CHR_DEV_SCH) += ch.o obj-$(CONFIG_SCSI_ENCLOSURE) += ses.o obj-$(CONFIG_SCSI_HISI_SAS) += hisi_sas/ +obj-$(CONFIG_VHBA) += vhba/ # This goes last, so that "real" scsi devices probe earlier obj-$(CONFIG_SCSI_DEBUG) += scsi_debug.o diff --git a/drivers/scsi/vhba/Kconfig b/drivers/scsi/vhba/Kconfig new file mode 100644 index 000000000000..e70a381fe3df --- /dev/null +++ b/drivers/scsi/vhba/Kconfig @@ -0,0 +1,9 @@ +config VHBA + tristate "Virtual (SCSI) Host Bus Adapter" + depends on SCSI + help + This is the in-kernel part of CDEmu, a CD/DVD-ROM device + emulator. + + This driver can also be built as a module. If so, the module + will be called vhba. diff --git a/drivers/scsi/vhba/Makefile b/drivers/scsi/vhba/Makefile new file mode 100644 index 000000000000..2d7524b66199 --- /dev/null +++ b/drivers/scsi/vhba/Makefile @@ -0,0 +1,4 @@ +VHBA_VERSION := 20240917 + +obj-$(CONFIG_VHBA) += vhba.o +ccflags-y := -DVHBA_VERSION=\"$(VHBA_VERSION)\" -Werror diff --git a/drivers/scsi/vhba/vhba.c b/drivers/scsi/vhba/vhba.c new file mode 100644 index 000000000000..878a3be0ba2b --- /dev/null +++ b/drivers/scsi/vhba/vhba.c @@ -0,0 +1,1132 @@ +/* + * vhba.c + * + * Copyright (C) 2007-2012 Chia-I Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define pr_fmt(fmt) "vhba: " fmt + +#include + +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#ifdef CONFIG_COMPAT +#include +#endif +#include +#include +#include +#include +#include +#include + + +MODULE_AUTHOR("Chia-I Wu"); +MODULE_VERSION(VHBA_VERSION); +MODULE_DESCRIPTION("Virtual SCSI HBA"); +MODULE_LICENSE("GPL"); + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#define sdev_dbg(sdev, fmt, a...) \ + dev_dbg(&(sdev)->sdev_gendev, fmt, ##a) +#define scmd_dbg(scmd, fmt, a...) \ + dev_dbg(&(scmd)->device->sdev_gendev, fmt, ##a) +#endif + +#define VHBA_MAX_SECTORS_PER_IO 256 +#define VHBA_MAX_BUS 16 +#define VHBA_MAX_ID 16 +#define VHBA_MAX_DEVICES (VHBA_MAX_BUS * (VHBA_MAX_ID-1)) +#define VHBA_KBUF_SIZE PAGE_SIZE + +#define DATA_TO_DEVICE(dir) ((dir) == DMA_TO_DEVICE || (dir) == DMA_BIDIRECTIONAL) +#define DATA_FROM_DEVICE(dir) ((dir) == DMA_FROM_DEVICE || (dir) == DMA_BIDIRECTIONAL) + + +static int vhba_can_queue = 32; +module_param_named(can_queue, vhba_can_queue, int, 0); + + +enum vhba_req_state { + VHBA_REQ_FREE, + VHBA_REQ_PENDING, + VHBA_REQ_READING, + VHBA_REQ_SENT, + VHBA_REQ_WRITING, +}; + +struct vhba_command { + struct scsi_cmnd *cmd; + /* metatags are per-host. not to be confused with + queue tags that are usually per-lun */ + unsigned long metatag; + int status; + struct list_head entry; +}; + +struct vhba_device { + unsigned int num; + spinlock_t cmd_lock; + struct list_head cmd_list; + wait_queue_head_t cmd_wq; + atomic_t refcnt; + + unsigned char *kbuf; + size_t kbuf_size; +}; + +struct vhba_host { + struct Scsi_Host *shost; + spinlock_t cmd_lock; + int cmd_next; + struct vhba_command *commands; + spinlock_t dev_lock; + struct vhba_device *devices[VHBA_MAX_DEVICES]; + int num_devices; + DECLARE_BITMAP(chgmap, VHBA_MAX_DEVICES); + int chgtype[VHBA_MAX_DEVICES]; + struct work_struct scan_devices; +}; + +#define MAX_COMMAND_SIZE 16 + +struct vhba_request { + __u32 metatag; + __u32 lun; + __u8 cdb[MAX_COMMAND_SIZE]; + __u8 cdb_len; + __u32 data_len; +}; + +struct vhba_response { + __u32 metatag; + __u32 status; + __u32 data_len; +}; + + + +static struct vhba_command *vhba_alloc_command (void); +static void vhba_free_command (struct vhba_command *vcmd); + +static struct platform_device vhba_platform_device; + + + +/* These functions define a symmetric 1:1 mapping between device numbers and + the bus and id. We have reserved the last id per bus for the host itself. */ +static void devnum_to_bus_and_id(unsigned int devnum, unsigned int *bus, unsigned int *id) +{ + *bus = devnum / (VHBA_MAX_ID-1); + *id = devnum % (VHBA_MAX_ID-1); +} + +static unsigned int bus_and_id_to_devnum(unsigned int bus, unsigned int id) +{ + return (bus * (VHBA_MAX_ID-1)) + id; +} + +static struct vhba_device *vhba_device_alloc (void) +{ + struct vhba_device *vdev; + + vdev = kzalloc(sizeof(struct vhba_device), GFP_KERNEL); + if (!vdev) { + return NULL; + } + + spin_lock_init(&vdev->cmd_lock); + INIT_LIST_HEAD(&vdev->cmd_list); + init_waitqueue_head(&vdev->cmd_wq); + atomic_set(&vdev->refcnt, 1); + + vdev->kbuf = NULL; + vdev->kbuf_size = 0; + + return vdev; +} + +static void vhba_device_put (struct vhba_device *vdev) +{ + if (atomic_dec_and_test(&vdev->refcnt)) { + kfree(vdev); + } +} + +static struct vhba_device *vhba_device_get (struct vhba_device *vdev) +{ + atomic_inc(&vdev->refcnt); + + return vdev; +} + +static int vhba_device_queue (struct vhba_device *vdev, struct scsi_cmnd *cmd) +{ + struct vhba_host *vhost; + struct vhba_command *vcmd; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + vcmd = vhba_alloc_command(); + if (!vcmd) { + return SCSI_MLQUEUE_HOST_BUSY; + } + + vcmd->cmd = cmd; + + spin_lock_irqsave(&vdev->cmd_lock, flags); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + vcmd->metatag = scsi_cmd_to_rq(vcmd->cmd)->tag; +#else + vcmd->metatag = vcmd->cmd->request->tag; +#endif + list_add_tail(&vcmd->entry, &vdev->cmd_list); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + wake_up_interruptible(&vdev->cmd_wq); + + return 0; +} + +static int vhba_device_dequeue (struct vhba_device *vdev, struct scsi_cmnd *cmd) +{ + struct vhba_command *vcmd; + int retval; + unsigned long flags; + + spin_lock_irqsave(&vdev->cmd_lock, flags); + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->cmd == cmd) { + list_del_init(&vcmd->entry); + break; + } + } + + /* command not found */ + if (&vcmd->entry == &vdev->cmd_list) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + return SUCCESS; + } + + while (vcmd->status == VHBA_REQ_READING || vcmd->status == VHBA_REQ_WRITING) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + scmd_dbg(cmd, "wait for I/O before aborting\n"); + schedule_timeout(1); + spin_lock_irqsave(&vdev->cmd_lock, flags); + } + + retval = (vcmd->status == VHBA_REQ_SENT) ? FAILED : SUCCESS; + + vhba_free_command(vcmd); + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return retval; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) +static int vhba_slave_alloc(struct scsi_device *sdev) +{ + struct Scsi_Host *shost = sdev->host; + + sdev_dbg(sdev, "enabling tagging (queue depth: %i).\n", sdev->queue_depth); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) + if (!shost_use_blk_mq(shost) && shost->bqt) { +#else + if (shost->bqt) { +#endif + blk_queue_init_tags(sdev->request_queue, sdev->queue_depth, shost->bqt); + } + scsi_adjust_queue_depth(sdev, 0, sdev->queue_depth); + + return 0; +} +#endif + +static void vhba_scan_devices_add (struct vhba_host *vhost, int bus, int id) +{ + struct scsi_device *sdev; + + sdev = scsi_device_lookup(vhost->shost, bus, id, 0); + if (!sdev) { + scsi_add_device(vhost->shost, bus, id, 0); + } else { + dev_warn(&vhost->shost->shost_gendev, "tried to add an already-existing device %d:%d:0!\n", bus, id); + scsi_device_put(sdev); + } +} + +static void vhba_scan_devices_remove (struct vhba_host *vhost, int bus, int id) +{ + struct scsi_device *sdev; + + sdev = scsi_device_lookup(vhost->shost, bus, id, 0); + if (sdev) { + scsi_remove_device(sdev); + scsi_device_put(sdev); + } else { + dev_warn(&vhost->shost->shost_gendev, "tried to remove non-existing device %d:%d:0!\n", bus, id); + } +} + +static void vhba_scan_devices (struct work_struct *work) +{ + struct vhba_host *vhost = container_of(work, struct vhba_host, scan_devices); + unsigned long flags; + int change, exists; + unsigned int devnum; + unsigned int bus, id; + + for (;;) { + spin_lock_irqsave(&vhost->dev_lock, flags); + + devnum = find_first_bit(vhost->chgmap, VHBA_MAX_DEVICES); + if (devnum >= VHBA_MAX_DEVICES) { + spin_unlock_irqrestore(&vhost->dev_lock, flags); + break; + } + change = vhost->chgtype[devnum]; + exists = vhost->devices[devnum] != NULL; + + vhost->chgtype[devnum] = 0; + clear_bit(devnum, vhost->chgmap); + + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + devnum_to_bus_and_id(devnum, &bus, &id); + + if (change < 0) { + dev_dbg(&vhost->shost->shost_gendev, "trying to remove target %d:%d:0\n", bus, id); + vhba_scan_devices_remove(vhost, bus, id); + } else if (change > 0) { + dev_dbg(&vhost->shost->shost_gendev, "trying to add target %d:%d:0\n", bus, id); + vhba_scan_devices_add(vhost, bus, id); + } else { + /* quick sequence of add/remove or remove/add; we determine + which one it was by checking if device structure exists */ + if (exists) { + /* remove followed by add: remove and (re)add */ + dev_dbg(&vhost->shost->shost_gendev, "trying to (re)add target %d:%d:0\n", bus, id); + vhba_scan_devices_remove(vhost, bus, id); + vhba_scan_devices_add(vhost, bus, id); + } else { + /* add followed by remove: no-op */ + dev_dbg(&vhost->shost->shost_gendev, "no-op for target %d:%d:0\n", bus, id); + } + } + } +} + +static int vhba_add_device (struct vhba_device *vdev) +{ + struct vhba_host *vhost; + unsigned int devnum; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + vhba_device_get(vdev); + + spin_lock_irqsave(&vhost->dev_lock, flags); + if (vhost->num_devices >= VHBA_MAX_DEVICES) { + spin_unlock_irqrestore(&vhost->dev_lock, flags); + vhba_device_put(vdev); + return -EBUSY; + } + + for (devnum = 0; devnum < VHBA_MAX_DEVICES; devnum++) { + if (vhost->devices[devnum] == NULL) { + vdev->num = devnum; + vhost->devices[devnum] = vdev; + vhost->num_devices++; + set_bit(devnum, vhost->chgmap); + vhost->chgtype[devnum]++; + break; + } + } + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + schedule_work(&vhost->scan_devices); + + return 0; +} + +static int vhba_remove_device (struct vhba_device *vdev) +{ + struct vhba_host *vhost; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->dev_lock, flags); + set_bit(vdev->num, vhost->chgmap); + vhost->chgtype[vdev->num]--; + vhost->devices[vdev->num] = NULL; + vhost->num_devices--; + spin_unlock_irqrestore(&vhost->dev_lock, flags); + + vhba_device_put(vdev); + + schedule_work(&vhost->scan_devices); + + return 0; +} + +static struct vhba_device *vhba_lookup_device (int devnum) +{ + struct vhba_host *vhost; + struct vhba_device *vdev = NULL; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + if (likely(devnum < VHBA_MAX_DEVICES)) { + spin_lock_irqsave(&vhost->dev_lock, flags); + vdev = vhost->devices[devnum]; + if (vdev) { + vdev = vhba_device_get(vdev); + } + + spin_unlock_irqrestore(&vhost->dev_lock, flags); + } + + return vdev; +} + +static struct vhba_command *vhba_alloc_command (void) +{ + struct vhba_host *vhost; + struct vhba_command *vcmd; + unsigned long flags; + int i; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->cmd_lock, flags); + + vcmd = vhost->commands + vhost->cmd_next++; + if (vcmd->status != VHBA_REQ_FREE) { + for (i = 0; i < vhba_can_queue; i++) { + vcmd = vhost->commands + i; + + if (vcmd->status == VHBA_REQ_FREE) { + vhost->cmd_next = i + 1; + break; + } + } + + if (i == vhba_can_queue) { + vcmd = NULL; + } + } + + if (vcmd) { + vcmd->status = VHBA_REQ_PENDING; + } + + vhost->cmd_next %= vhba_can_queue; + + spin_unlock_irqrestore(&vhost->cmd_lock, flags); + + return vcmd; +} + +static void vhba_free_command (struct vhba_command *vcmd) +{ + struct vhba_host *vhost; + unsigned long flags; + + vhost = platform_get_drvdata(&vhba_platform_device); + + spin_lock_irqsave(&vhost->cmd_lock, flags); + vcmd->status = VHBA_REQ_FREE; + spin_unlock_irqrestore(&vhost->cmd_lock, flags); +} + +static int vhba_queuecommand (struct Scsi_Host *shost, struct scsi_cmnd *cmd) +{ + struct vhba_device *vdev; + int retval; + unsigned int devnum; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) + scmd_dbg(cmd, "queue %p tag %i\n", cmd, scsi_cmd_to_rq(cmd)->tag); +#else + scmd_dbg(cmd, "queue %p tag %i\n", cmd, cmd->request->tag); +#endif + + devnum = bus_and_id_to_devnum(cmd->device->channel, cmd->device->id); + vdev = vhba_lookup_device(devnum); + if (!vdev) { + scmd_dbg(cmd, "no such device\n"); + + cmd->result = DID_NO_CONNECT << 16; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(cmd); +#else + cmd->scsi_done(cmd); +#endif + + return 0; + } + + retval = vhba_device_queue(vdev, cmd); + + vhba_device_put(vdev); + + return retval; +} + +static int vhba_abort (struct scsi_cmnd *cmd) +{ + struct vhba_device *vdev; + int retval = SUCCESS; + unsigned int devnum; + + scmd_dbg(cmd, "abort %p\n", cmd); + + devnum = bus_and_id_to_devnum(cmd->device->channel, cmd->device->id); + vdev = vhba_lookup_device(devnum); + if (vdev) { + retval = vhba_device_dequeue(vdev, cmd); + vhba_device_put(vdev); + } else { + cmd->result = DID_NO_CONNECT << 16; + } + + return retval; +} + +static struct scsi_host_template vhba_template = { + .module = THIS_MODULE, + .name = "vhba", + .proc_name = "vhba", + .queuecommand = vhba_queuecommand, + .eh_abort_handler = vhba_abort, + .this_id = -1, + .max_sectors = VHBA_MAX_SECTORS_PER_IO, + .sg_tablesize = 256, +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) + .slave_alloc = vhba_slave_alloc, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(6, 14, 0) + .tag_alloc_policy = BLK_TAG_ALLOC_RR, +#else + .tag_alloc_policy_rr = true, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + .use_blk_tags = 1, +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) + .max_segment_size = VHBA_KBUF_SIZE, +#endif +}; + +static ssize_t do_request (struct vhba_device *vdev, unsigned long metatag, struct scsi_cmnd *cmd, char __user *buf, size_t buf_len) +{ + struct vhba_request vreq; + ssize_t ret; + + scmd_dbg(cmd, "request %lu (%p), cdb 0x%x, bufflen %d, sg count %d\n", + metatag, cmd, cmd->cmnd[0], scsi_bufflen(cmd), scsi_sg_count(cmd)); + + ret = sizeof(vreq); + if (DATA_TO_DEVICE(cmd->sc_data_direction)) { + ret += scsi_bufflen(cmd); + } + + if (ret > buf_len) { + scmd_dbg(cmd, "buffer too small (%zd < %zd) for a request\n", buf_len, ret); + return -EIO; + } + + vreq.metatag = metatag; + vreq.lun = cmd->device->lun; + memcpy(vreq.cdb, cmd->cmnd, MAX_COMMAND_SIZE); + vreq.cdb_len = cmd->cmd_len; + vreq.data_len = scsi_bufflen(cmd); + + if (copy_to_user(buf, &vreq, sizeof(vreq))) { + return -EFAULT; + } + + if (DATA_TO_DEVICE(cmd->sc_data_direction) && vreq.data_len) { + buf += sizeof(vreq); + + if (scsi_sg_count(cmd)) { + unsigned char *kaddr, *uaddr; + struct scatterlist *sglist = scsi_sglist(cmd); + struct scatterlist *sg; + int i; + + uaddr = (unsigned char *) buf; + + for_each_sg(sglist, sg, scsi_sg_count(cmd), i) { + size_t len = sg->length; + + if (len > vdev->kbuf_size) { + scmd_dbg(cmd, "segment size (%zu) exceeds kbuf size (%zu)!", len, vdev->kbuf_size); + len = vdev->kbuf_size; + } + + kaddr = kmap_atomic(sg_page(sg)); + memcpy(vdev->kbuf, kaddr + sg->offset, len); + kunmap_atomic(kaddr); + + if (copy_to_user(uaddr, vdev->kbuf, len)) { + return -EFAULT; + } + uaddr += len; + } + } else { + if (copy_to_user(buf, scsi_sglist(cmd), vreq.data_len)) { + return -EFAULT; + } + } + } + + return ret; +} + +static ssize_t do_response (struct vhba_device *vdev, unsigned long metatag, struct scsi_cmnd *cmd, const char __user *buf, size_t buf_len, struct vhba_response *res) +{ + ssize_t ret = 0; + + scmd_dbg(cmd, "response %lu (%p), status %x, data len %d, sg count %d\n", + metatag, cmd, res->status, res->data_len, scsi_sg_count(cmd)); + + if (res->status) { + if (res->data_len > SCSI_SENSE_BUFFERSIZE) { + scmd_dbg(cmd, "truncate sense (%d < %d)", SCSI_SENSE_BUFFERSIZE, res->data_len); + res->data_len = SCSI_SENSE_BUFFERSIZE; + } + + if (copy_from_user(cmd->sense_buffer, buf, res->data_len)) { + return -EFAULT; + } + + cmd->result = res->status; + + ret += res->data_len; + } else if (DATA_FROM_DEVICE(cmd->sc_data_direction) && scsi_bufflen(cmd)) { + size_t to_read; + + if (res->data_len > scsi_bufflen(cmd)) { + scmd_dbg(cmd, "truncate data (%d < %d)\n", scsi_bufflen(cmd), res->data_len); + res->data_len = scsi_bufflen(cmd); + } + + to_read = res->data_len; + + if (scsi_sg_count(cmd)) { + unsigned char *kaddr, *uaddr; + struct scatterlist *sglist = scsi_sglist(cmd); + struct scatterlist *sg; + int i; + + uaddr = (unsigned char *)buf; + + for_each_sg(sglist, sg, scsi_sg_count(cmd), i) { + size_t len = (sg->length < to_read) ? sg->length : to_read; + + if (len > vdev->kbuf_size) { + scmd_dbg(cmd, "segment size (%zu) exceeds kbuf size (%zu)!", len, vdev->kbuf_size); + len = vdev->kbuf_size; + } + + if (copy_from_user(vdev->kbuf, uaddr, len)) { + return -EFAULT; + } + uaddr += len; + + kaddr = kmap_atomic(sg_page(sg)); + memcpy(kaddr + sg->offset, vdev->kbuf, len); + kunmap_atomic(kaddr); + + to_read -= len; + if (to_read == 0) { + break; + } + } + } else { + if (copy_from_user(scsi_sglist(cmd), buf, res->data_len)) { + return -EFAULT; + } + + to_read -= res->data_len; + } + + scsi_set_resid(cmd, to_read); + + ret += res->data_len - to_read; + } + + return ret; +} + +static struct vhba_command *next_command (struct vhba_device *vdev) +{ + struct vhba_command *vcmd; + + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->status == VHBA_REQ_PENDING) { + break; + } + } + + if (&vcmd->entry == &vdev->cmd_list) { + vcmd = NULL; + } + + return vcmd; +} + +static struct vhba_command *match_command (struct vhba_device *vdev, __u32 metatag) +{ + struct vhba_command *vcmd; + + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + if (vcmd->metatag == metatag) { + break; + } + } + + if (&vcmd->entry == &vdev->cmd_list) { + vcmd = NULL; + } + + return vcmd; +} + +static struct vhba_command *wait_command (struct vhba_device *vdev, unsigned long flags) +{ + struct vhba_command *vcmd; + DEFINE_WAIT(wait); + + while (!(vcmd = next_command(vdev))) { + if (signal_pending(current)) { + break; + } + + prepare_to_wait(&vdev->cmd_wq, &wait, TASK_INTERRUPTIBLE); + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + schedule(); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + } + + finish_wait(&vdev->cmd_wq, &wait); + if (vcmd) { + vcmd->status = VHBA_REQ_READING; + } + + return vcmd; +} + +static ssize_t vhba_ctl_read (struct file *file, char __user *buf, size_t buf_len, loff_t *offset) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + ssize_t ret; + unsigned long flags; + + vdev = file->private_data; + + /* Get next command */ + if (file->f_flags & O_NONBLOCK) { + /* Non-blocking variant */ + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = next_command(vdev); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + if (!vcmd) { + return -EWOULDBLOCK; + } + } else { + /* Blocking variant */ + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = wait_command(vdev, flags); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + if (!vcmd) { + return -ERESTARTSYS; + } + } + + ret = do_request(vdev, vcmd->metatag, vcmd->cmd, buf, buf_len); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (ret >= 0) { + vcmd->status = VHBA_REQ_SENT; + *offset += ret; + } else { + vcmd->status = VHBA_REQ_PENDING; + } + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return ret; +} + +static ssize_t vhba_ctl_write (struct file *file, const char __user *buf, size_t buf_len, loff_t *offset) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + struct vhba_response res; + ssize_t ret; + unsigned long flags; + + if (buf_len < sizeof(res)) { + return -EIO; + } + + if (copy_from_user(&res, buf, sizeof(res))) { + return -EFAULT; + } + + vdev = file->private_data; + + spin_lock_irqsave(&vdev->cmd_lock, flags); + vcmd = match_command(vdev, res.metatag); + if (!vcmd || vcmd->status != VHBA_REQ_SENT) { + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + pr_debug("ctl dev #%u not expecting response\n", vdev->num); + return -EIO; + } + vcmd->status = VHBA_REQ_WRITING; + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + ret = do_response(vdev, vcmd->metatag, vcmd->cmd, buf + sizeof(res), buf_len - sizeof(res), &res); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (ret >= 0) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(vcmd->cmd); +#else + vcmd->cmd->scsi_done(vcmd->cmd); +#endif + ret += sizeof(res); + + /* don't compete with vhba_device_dequeue */ + if (!list_empty(&vcmd->entry)) { + list_del_init(&vcmd->entry); + vhba_free_command(vcmd); + } + } else { + vcmd->status = VHBA_REQ_SENT; + } + + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return ret; +} + +static long vhba_ctl_ioctl (struct file *file, unsigned int cmd, unsigned long arg) +{ + struct vhba_device *vdev = file->private_data; + struct vhba_host *vhost = platform_get_drvdata(&vhba_platform_device); + + switch (cmd) { + case 0xBEEF001: { + unsigned int ident[4]; /* host, channel, id, lun */ + + ident[0] = vhost->shost->host_no; + devnum_to_bus_and_id(vdev->num, &ident[1], &ident[2]); + ident[3] = 0; /* lun */ + + if (copy_to_user((void *) arg, ident, sizeof(ident))) { + return -EFAULT; + } + + return 0; + } + case 0xBEEF002: { + unsigned int devnum = vdev->num; + + if (copy_to_user((void *) arg, &devnum, sizeof(devnum))) { + return -EFAULT; + } + + return 0; + } + } + + return -ENOTTY; +} + +#ifdef CONFIG_COMPAT +static long vhba_ctl_compat_ioctl (struct file *file, unsigned int cmd, unsigned long arg) +{ + unsigned long compat_arg = (unsigned long)compat_ptr(arg); + return vhba_ctl_ioctl(file, cmd, compat_arg); +} +#endif + +static unsigned int vhba_ctl_poll (struct file *file, poll_table *wait) +{ + struct vhba_device *vdev = file->private_data; + unsigned int mask = 0; + unsigned long flags; + + poll_wait(file, &vdev->cmd_wq, wait); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + if (next_command(vdev)) { + mask |= POLLIN | POLLRDNORM; + } + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + return mask; +} + +static int vhba_ctl_open (struct inode *inode, struct file *file) +{ + struct vhba_device *vdev; + int retval; + + pr_debug("ctl dev open\n"); + + /* check if vhba is probed */ + if (!platform_get_drvdata(&vhba_platform_device)) { + return -ENODEV; + } + + vdev = vhba_device_alloc(); + if (!vdev) { + return -ENOMEM; + } + + vdev->kbuf_size = VHBA_KBUF_SIZE; + vdev->kbuf = kzalloc(vdev->kbuf_size, GFP_KERNEL); + if (!vdev->kbuf) { + return -ENOMEM; + } + + if (!(retval = vhba_add_device(vdev))) { + file->private_data = vdev; + } + + vhba_device_put(vdev); + + return retval; +} + +static int vhba_ctl_release (struct inode *inode, struct file *file) +{ + struct vhba_device *vdev; + struct vhba_command *vcmd; + unsigned long flags; + + vdev = file->private_data; + + pr_debug("ctl dev release\n"); + + vhba_device_get(vdev); + vhba_remove_device(vdev); + + spin_lock_irqsave(&vdev->cmd_lock, flags); + list_for_each_entry(vcmd, &vdev->cmd_list, entry) { + WARN_ON(vcmd->status == VHBA_REQ_READING || vcmd->status == VHBA_REQ_WRITING); + + scmd_dbg(vcmd->cmd, "device released with command %lu (%p)\n", vcmd->metatag, vcmd->cmd); + vcmd->cmd->result = DID_NO_CONNECT << 16; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) + scsi_done(vcmd->cmd); +#else + vcmd->cmd->scsi_done(vcmd->cmd); +#endif + vhba_free_command(vcmd); + } + INIT_LIST_HEAD(&vdev->cmd_list); + spin_unlock_irqrestore(&vdev->cmd_lock, flags); + + kfree(vdev->kbuf); + vdev->kbuf = NULL; + + vhba_device_put(vdev); + + return 0; +} + +static struct file_operations vhba_ctl_fops = { + .owner = THIS_MODULE, + .open = vhba_ctl_open, + .release = vhba_ctl_release, + .read = vhba_ctl_read, + .write = vhba_ctl_write, + .poll = vhba_ctl_poll, + .unlocked_ioctl = vhba_ctl_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = vhba_ctl_compat_ioctl, +#endif +}; + +static struct miscdevice vhba_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "vhba_ctl", + .fops = &vhba_ctl_fops, +}; + +static int vhba_probe (struct platform_device *pdev) +{ + struct Scsi_Host *shost; + struct vhba_host *vhost; + int i; + + vhba_can_queue = clamp(vhba_can_queue, 1, 256); + + shost = scsi_host_alloc(&vhba_template, sizeof(struct vhba_host)); + if (!shost) { + return -ENOMEM; + } + + shost->max_channel = VHBA_MAX_BUS-1; + shost->max_id = VHBA_MAX_ID; + /* we don't support lun > 0 */ + shost->max_lun = 1; + shost->max_cmd_len = MAX_COMMAND_SIZE; + shost->can_queue = vhba_can_queue; + shost->cmd_per_lun = vhba_can_queue; + + vhost = (struct vhba_host *)shost->hostdata; + memset(vhost, 0, sizeof(struct vhba_host)); + + vhost->shost = shost; + vhost->num_devices = 0; + spin_lock_init(&vhost->dev_lock); + spin_lock_init(&vhost->cmd_lock); + INIT_WORK(&vhost->scan_devices, vhba_scan_devices); + vhost->cmd_next = 0; + vhost->commands = kzalloc(vhba_can_queue * sizeof(struct vhba_command), GFP_KERNEL); + if (!vhost->commands) { + return -ENOMEM; + } + + for (i = 0; i < vhba_can_queue; i++) { + vhost->commands[i].status = VHBA_REQ_FREE; + } + + platform_set_drvdata(pdev, vhost); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + i = scsi_init_shared_tag_map(shost, vhba_can_queue); + if (i) return i; +#endif + + if (scsi_add_host(shost, &pdev->dev)) { + scsi_host_put(shost); + return -ENOMEM; + } + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) +static int vhba_remove (struct platform_device *pdev) +#else +static void vhba_remove (struct platform_device *pdev) +#endif +{ + struct vhba_host *vhost; + struct Scsi_Host *shost; + + vhost = platform_get_drvdata(pdev); + shost = vhost->shost; + + scsi_remove_host(shost); + scsi_host_put(shost); + + kfree(vhost->commands); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + return 0; +#endif +} + +static void vhba_release (struct device * dev) +{ + return; +} + +static struct platform_device vhba_platform_device = { + .name = "vhba", + .id = -1, + .dev = { + .release = vhba_release, + }, +}; + +static struct platform_driver vhba_platform_driver = { + .driver = { + .owner = THIS_MODULE, + .name = "vhba", + }, + .probe = vhba_probe, + .remove = vhba_remove, +}; + +static int __init vhba_init (void) +{ + int ret; + + ret = platform_device_register(&vhba_platform_device); + if (ret < 0) { + return ret; + } + + ret = platform_driver_register(&vhba_platform_driver); + if (ret < 0) { + platform_device_unregister(&vhba_platform_device); + return ret; + } + + ret = misc_register(&vhba_miscdev); + if (ret < 0) { + platform_driver_unregister(&vhba_platform_driver); + platform_device_unregister(&vhba_platform_device); + return ret; + } + + return 0; +} + +static void __exit vhba_exit(void) +{ + misc_deregister(&vhba_miscdev); + platform_driver_unregister(&vhba_platform_driver); + platform_device_unregister(&vhba_platform_device); +} + +module_init(vhba_init); +module_exit(vhba_exit); + diff --git a/fs/select.c b/fs/select.c index 7da531b1cf6b..0eaf3522abe9 100644 --- a/fs/select.c +++ b/fs/select.c @@ -857,7 +857,7 @@ static inline __poll_t do_pollfd(struct pollfd *pollfd, poll_table *pwait, int fd = pollfd->fd; __poll_t mask, filter; - if (fd < 0) + if (unlikely(fd < 0)) return 0; CLASS(fd, f)(fd); diff --git a/include/linux/mm.h b/include/linux/mm.h index fdda6b16263b..e63d17960c53 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -193,6 +193,14 @@ static inline void __mm_zero_struct_page(struct page *page) extern int sysctl_max_map_count; +extern bool sysctl_workingset_protection; +extern u8 sysctl_anon_min_ratio; +extern u8 sysctl_clean_low_ratio; +extern u8 sysctl_clean_min_ratio; +int vm_workingset_protection_update_handler( + const struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + extern unsigned long sysctl_user_reserve_kbytes; extern unsigned long sysctl_admin_reserve_kbytes; diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 26baa78f1ca7..8fa2ff2682bc 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1341,7 +1341,7 @@ struct readahead_control { ._index = i, \ } -#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE) +#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE) void page_cache_ra_unbounded(struct readahead_control *, unsigned long nr_to_read, unsigned long lookahead_count); diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index a0bb6d012137..93129fea552e 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -168,6 +168,8 @@ static inline void set_userns_rlimit_max(struct user_namespace *ns, #ifdef CONFIG_USER_NS +extern int unprivileged_userns_clone; + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { if (ns) @@ -201,6 +203,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns); struct ns_common *ns_get_owner(struct ns_common *ns); #else +#define unprivileged_userns_clone 0 + static inline struct user_namespace *get_user_ns(struct user_namespace *ns) { return &init_user_ns; diff --git a/include/linux/wait.h b/include/linux/wait.h index 965a19809c7e..3d442267a256 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -163,6 +163,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head) extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); +extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); @@ -1195,6 +1196,7 @@ do { \ */ void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); +void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state); void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); diff --git a/init/Kconfig b/init/Kconfig index bf3a920064be..c524858118d2 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -163,6 +163,10 @@ config THREAD_INFO_IN_TASK menu "General setup" +config CACHY + bool "Some kernel tweaks by CachyOS" + default y + config BROKEN bool @@ -1330,6 +1334,22 @@ config USER_NS If unsure, say N. +config USER_NS_UNPRIVILEGED + bool "Allow unprivileged users to create namespaces" + default y + depends on USER_NS + help + When disabled, unprivileged users will not be able to create + new namespaces. Allowing users to create their own namespaces + has been part of several recent local privilege escalation + exploits, so if you need user namespaces but are + paranoid^Wsecurity-conscious you want to disable this. + + This setting can be overridden at runtime via the + kernel.unprivileged_userns_clone sysctl. + + If unsure, say Y. + config PID_NS bool "PID Namespaces" default y @@ -1479,6 +1499,12 @@ config CC_OPTIMIZE_FOR_PERFORMANCE with the "-O2" compiler flag for best performance and most helpful compile-time warnings. +config CC_OPTIMIZE_FOR_PERFORMANCE_O3 + bool "Optimize more for performance (-O3)" + help + Choosing this option will pass "-O3" to your compiler to optimize + the kernel yet more for performance. + config CC_OPTIMIZE_FOR_SIZE bool "Optimize for size (-Os)" help diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz index ce1435cb08b1..e1359db5561e 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz @@ -40,6 +40,27 @@ choice on SMP and NUMA systems and exactly dividing by both PAL and NTSC frame rates for video and multimedia work. + config HZ_500 + bool "500 HZ" + help + 500 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_600 + bool "600 HZ" + help + 600 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + + config HZ_750 + bool "750 HZ" + help + 750 Hz is a balanced timer frequency. Provides fast interactivity + on desktops with good smoothness without increasing CPU power + consumption and sacrificing the battery life on laptops. + config HZ_1000 bool "1000 HZ" help @@ -53,6 +74,9 @@ config HZ default 100 if HZ_100 default 250 if HZ_250 default 300 if HZ_300 + default 500 if HZ_500 + default 600 if HZ_600 + default 750 if HZ_750 default 1000 if HZ_1000 config SCHED_HRTICK diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 54ea59ff8fbe..18f87e0dd137 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -88,7 +88,7 @@ endchoice config PREEMPT_RT bool "Fully Preemptible Kernel (Real-Time)" - depends on EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST + depends on ARCH_SUPPORTS_RT && !COMPILE_TEST select PREEMPTION help This option turns the kernel into a real-time kernel by replacing diff --git a/kernel/fork.c b/kernel/fork.c index 168681fc4b25..39e47bfaea58 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -106,6 +106,10 @@ #include #include +#ifdef CONFIG_USER_NS +#include +#endif + #include #include #include @@ -2194,6 +2198,10 @@ __latent_entropy struct task_struct *copy_process( if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) return ERR_PTR(-EINVAL); + if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) + if (!capable(CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + /* * Thread groups must share signals as well, and detached threads * can only be started up within the thread group. @@ -3354,6 +3362,12 @@ int ksys_unshare(unsigned long unshare_flags) if (unshare_flags & CLONE_NEWNS) unshare_flags |= CLONE_FS; + if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) { + err = -EPERM; + if (!capable(CAP_SYS_ADMIN)) + goto bad_unshare_out; + } + err = check_unshare_flags(unshare_flags); if (err) goto bad_unshare_out; diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 2ddb827e3bea..464049c4af3f 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -747,6 +747,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) struct task_struct *new, *owner; unsigned long flags, new_flags; enum owner_state state; + int i = 0; lockdep_assert_preemption_disabled(); @@ -783,7 +784,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem) break; } - cpu_relax(); + if (i++ > 1000) + cpu_relax(); } return state; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8d0f462e8c8b..6122df2b471a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -76,10 +76,19 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; * * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ +#ifdef CONFIG_CACHY +unsigned int sysctl_sched_base_slice = 350000ULL; +static unsigned int normalized_sysctl_sched_base_slice = 350000ULL; +#else unsigned int sysctl_sched_base_slice = 700000ULL; static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; +#endif /* CONFIG_CACHY */ +#ifdef CONFIG_CACHY +__read_mostly unsigned int sysctl_sched_migration_cost = 300000UL; +#else __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; +#endif static int __init setup_sched_thermal_decay_shift(char *str) { @@ -124,8 +133,12 @@ int __weak arch_asym_cpu_priority(int cpu) * * (default: 5 msec, units: microseconds) */ +#ifdef CONFIG_CACHY +static unsigned int sysctl_sched_cfs_bandwidth_slice = 3000UL; +#else static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +#endif #ifdef CONFIG_NUMA_BALANCING /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 47972f34ea70..0892942cf913 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2790,7 +2790,7 @@ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); extern void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags); -#ifdef CONFIG_PREEMPT_RT +#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_CACHY) # define SCHED_NR_MIGRATE_BREAK 8 #else # define SCHED_NR_MIGRATE_BREAK 32 diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 51e38f5f4701..c5cc616484ba 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -47,6 +47,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_ } EXPORT_SYMBOL_GPL(add_wait_queue_priority); +void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) +{ + unsigned long flags; + + wq_entry->flags |= WQ_FLAG_EXCLUSIVE; + spin_lock_irqsave(&wq_head->lock, flags); + __add_wait_queue(wq_head, wq_entry); + spin_unlock_irqrestore(&wq_head->lock, flags); +} +EXPORT_SYMBOL(add_wait_queue_exclusive_lifo); + void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry) { unsigned long flags; @@ -258,6 +269,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent } EXPORT_SYMBOL(prepare_to_wait_exclusive); +void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state) +{ + unsigned long flags; + + wq_entry->flags |= WQ_FLAG_EXCLUSIVE; + spin_lock_irqsave(&wq_head->lock, flags); + if (list_empty(&wq_entry->entry)) + __add_wait_queue(wq_head, wq_entry); + set_current_state(state); + spin_unlock_irqrestore(&wq_head->lock, flags); +} +EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo); + void init_wait_entry(struct wait_queue_entry *wq_entry, int flags) { wq_entry->flags = flags; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 3b7a7308e35b..fe7bec454345 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -70,6 +70,9 @@ #ifdef CONFIG_RT_MUTEXES #include #endif +#ifdef CONFIG_USER_NS +#include +#endif /* shared constants to be used in various sysctls */ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; @@ -1595,6 +1598,15 @@ static const struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, +#ifdef CONFIG_USER_NS + { + .procname = "unprivileged_userns_clone", + .data = &unprivileged_userns_clone, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #ifdef CONFIG_PROC_SYSCTL { .procname = "tainted", diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 682f40d5632d..434a25f7b2ed 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -22,6 +22,13 @@ #include #include +/* sysctl */ +#ifdef CONFIG_USER_NS_UNPRIVILEGED +int unprivileged_userns_clone = 1; +#else +int unprivileged_userns_clone; +#endif + static struct kmem_cache *user_ns_cachep __ro_after_init; static DEFINE_MUTEX(userns_state_mutex); diff --git a/mm/Kconfig b/mm/Kconfig index e113f713b493..5fc1934a3095 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -462,6 +462,69 @@ config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP config ARCH_WANT_HUGETLB_VMEMMAP_PREINIT bool +config ANON_MIN_RATIO + int "Default value for vm.anon_min_ratio" + depends on SYSCTL + range 0 100 + default 15 + help + This option sets the default value for vm.anon_min_ratio sysctl knob. + + The vm.anon_min_ratio sysctl knob provides *hard* protection of + anonymous pages. The anonymous pages on the current node won't be + reclaimed under any conditions when their amount is below + vm.anon_min_ratio. This knob may be used to prevent excessive swap + thrashing when anonymous memory is low (for example, when memory is + going to be overfilled by compressed data of zram module). + + Setting this value too high (close to MemTotal) can result in + inability to swap and can lead to early OOM under memory pressure. + +config CLEAN_LOW_RATIO + int "Default value for vm.clean_low_ratio" + depends on SYSCTL + range 0 100 + default 0 + help + This option sets the default value for vm.clean_low_ratio sysctl knob. + + The vm.clean_low_ratio sysctl knob provides *best-effort* + protection of clean file pages. The file pages on the current node + won't be reclaimed under memory pressure when the amount of clean file + pages is below vm.clean_low_ratio *unless* we threaten to OOM. + Protection of clean file pages using this knob may be used when + swapping is still possible to + - prevent disk I/O thrashing under memory pressure; + - improve performance in disk cache-bound tasks under memory + pressure. + + Setting it to a high value may result in a early eviction of anonymous + pages into the swap space by attempting to hold the protected amount + of clean file pages in memory. + +config CLEAN_MIN_RATIO + int "Default value for vm.clean_min_ratio" + depends on SYSCTL + range 0 100 + default 15 + help + This option sets the default value for vm.clean_min_ratio sysctl knob. + + The vm.clean_min_ratio sysctl knob provides *hard* protection of + clean file pages. The file pages on the current node won't be + reclaimed under memory pressure when the amount of clean file pages is + below vm.clean_min_ratio. Hard protection of clean file pages using + this knob may be used to + - prevent disk I/O thrashing under memory pressure even with no free + swap space; + - improve performance in disk cache-bound tasks under memory + pressure; + - avoid high latency and prevent livelock in near-OOM conditions. + + Setting it to a high value may result in a early out-of-memory condition + due to the inability to reclaim the protected amount of clean file pages + when other types of pages cannot be reclaimed. + config HAVE_MEMBLOCK_PHYS_MAP bool @@ -654,7 +717,7 @@ config COMPACTION config COMPACT_UNEVICTABLE_DEFAULT int depends on COMPACTION - default 0 if PREEMPT_RT + default 0 if PREEMPT_RT || CACHY default 1 # diff --git a/mm/compaction.c b/mm/compaction.c index ca71fd3c3181..2070b1e5106f 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1923,7 +1923,11 @@ static int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNE * aggressively the kernel should compact memory in the * background. It takes values in the range [0, 100]. */ +#ifdef CONFIG_CACHY +static unsigned int __read_mostly sysctl_compaction_proactiveness; +#else static unsigned int __read_mostly sysctl_compaction_proactiveness = 20; +#endif static int sysctl_extfrag_threshold = 500; static int __read_mostly sysctl_compact_memory; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 47d76d03ce30..48615051a8cf 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -64,7 +64,11 @@ unsigned long transparent_hugepage_flags __read_mostly = #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE (1<> (20 - PAGE_SHIFT); /* Use a smaller cluster for small-memory machines */ @@ -1098,6 +1102,7 @@ void __init swap_setup(void) page_cluster = 2; else page_cluster = 3; +#endif /* CONFIG_CACHY */ /* * Right now other parts of the system means that we * _really_ don't want to cluster much more diff --git a/mm/util.c b/mm/util.c index 448117da071f..b9334775c51d 100644 --- a/mm/util.c +++ b/mm/util.c @@ -857,6 +857,40 @@ static const struct ctl_table util_sysctl_table[] = { .mode = 0644, .proc_handler = proc_doulongvec_minmax, }, + { + .procname = "workingset_protection", + .data = &sysctl_workingset_protection, + .maxlen = sizeof(bool), + .mode = 0644, + .proc_handler = &proc_dobool, + }, + { + .procname = "anon_min_ratio", + .data = &sysctl_anon_min_ratio, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = &vm_workingset_protection_update_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "clean_low_ratio", + .data = &sysctl_clean_low_ratio, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = &vm_workingset_protection_update_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "clean_min_ratio", + .data = &sysctl_clean_min_ratio, + .maxlen = sizeof(u8), + .mode = 0644, + .proc_handler = &vm_workingset_protection_update_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, }; static int __init init_vm_util_sysctls(void) diff --git a/mm/vmpressure.c b/mm/vmpressure.c index bd5183dfd879..3a410f53a07c 100644 --- a/mm/vmpressure.c +++ b/mm/vmpressure.c @@ -43,7 +43,11 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; * essence, they are percents: the higher the value, the more number * unsuccessful reclaims there were. */ +#ifdef CONFIG_CACHY +static const unsigned int vmpressure_level_med = 65; +#else static const unsigned int vmpressure_level_med = 60; +#endif static const unsigned int vmpressure_level_critical = 95; /* diff --git a/mm/vmscan.c b/mm/vmscan.c index 3783e45bfc92..4f991a6ae689 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -148,6 +148,15 @@ struct scan_control { /* The file folios on the current node are dangerously low */ unsigned int file_is_tiny:1; + /* The anonymous pages on the current node are below vm.anon_min_ratio */ + unsigned int anon_below_min:1; + + /* The clean file pages on the current node are below vm.clean_low_ratio */ + unsigned int clean_below_low:1; + + /* The clean file pages on the current node are below vm.clean_min_ratio */ + unsigned int clean_below_min:1; + /* Always discard instead of demoting to lower tier memory */ unsigned int no_demotion:1; @@ -197,10 +206,23 @@ struct scan_control { #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) #endif +bool sysctl_workingset_protection __read_mostly = true; +u8 sysctl_anon_min_ratio __read_mostly = CONFIG_ANON_MIN_RATIO; +u8 sysctl_clean_low_ratio __read_mostly = CONFIG_CLEAN_LOW_RATIO; +u8 sysctl_clean_min_ratio __read_mostly = CONFIG_CLEAN_MIN_RATIO; +static u64 sysctl_anon_min_ratio_kb __read_mostly = 0; +static u64 sysctl_clean_low_ratio_kb __read_mostly = 0; +static u64 sysctl_clean_min_ratio_kb __read_mostly = 0; +static u64 workingset_protection_prev_totalram __read_mostly = 0; + /* * From 0 .. MAX_SWAPPINESS. Higher means more swappy. */ +#ifdef CONFIG_CACHY +int vm_swappiness = 100; +#else int vm_swappiness = 60; +#endif #ifdef CONFIG_MEMCG @@ -1147,6 +1169,10 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, if (!sc->may_unmap && folio_mapped(folio)) goto keep_locked; + if (folio_is_file_lru(folio) ? sc->clean_below_min : + (sc->anon_below_min && !sc->clean_below_min)) + goto keep_locked; + /* * The number of dirty pages determines if a node is marked * reclaim_congested. kswapd will stall and start writing @@ -2521,6 +2547,15 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, goto out; } + /* + * Force-scan anon if clean file pages is under vm.clean_low_ratio + * or vm.clean_min_ratio. + */ + if (sc->clean_below_low || sc->clean_below_min) { + scan_balance = SCAN_ANON; + goto out; + } + /* * If there is enough inactive page cache, we do not reclaim * anything from the anonymous working right now. @@ -2638,6 +2673,14 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, BUG(); } + /* + * Hard protection of the working set. + * Don't reclaim anon/file pages when the amount is + * below the watermark of the same type. + */ + if (file ? sc->clean_below_min : sc->anon_below_min) + scan = 0; + nr[lru] = scan; } } @@ -2657,6 +2700,96 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, return can_demote(pgdat->node_id, sc); } +int vm_workingset_protection_update_handler(const struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); + if (ret || !write) + return ret; + + workingset_protection_prev_totalram = 0; + + return 0; +} + +static void prepare_workingset_protection(pg_data_t *pgdat, struct scan_control *sc) +{ + unsigned long node_mem_total; + struct sysinfo i; + + if (!(sysctl_workingset_protection)) { + sc->anon_below_min = 0; + sc->clean_below_low = 0; + sc->clean_below_min = 0; + return; + } + + if (likely(sysctl_anon_min_ratio || + sysctl_clean_low_ratio || + sysctl_clean_min_ratio)) { +#ifdef CONFIG_NUMA + si_meminfo_node(&i, pgdat->node_id); +#else //CONFIG_NUMA + si_meminfo(&i); +#endif //CONFIG_NUMA + node_mem_total = i.totalram; + + if (unlikely(workingset_protection_prev_totalram != node_mem_total)) { + sysctl_anon_min_ratio_kb = + node_mem_total * sysctl_anon_min_ratio / 100; + sysctl_clean_low_ratio_kb = + node_mem_total * sysctl_clean_low_ratio / 100; + sysctl_clean_min_ratio_kb = + node_mem_total * sysctl_clean_min_ratio / 100; + workingset_protection_prev_totalram = node_mem_total; + } + } + + /* + * Check the number of anonymous pages to protect them from + * reclaiming if their amount is below the specified. + */ + if (sysctl_anon_min_ratio) { + unsigned long reclaimable_anon; + + reclaimable_anon = + node_page_state(pgdat, NR_ACTIVE_ANON) + + node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_ISOLATED_ANON); + + sc->anon_below_min = reclaimable_anon < sysctl_anon_min_ratio_kb; + } else + sc->anon_below_min = 0; + + /* + * Check the number of clean file pages to protect them from + * reclaiming if their amount is below the specified. + */ + if (sysctl_clean_low_ratio || sysctl_clean_min_ratio) { + unsigned long reclaimable_file, dirty, clean; + + reclaimable_file = + node_page_state(pgdat, NR_ACTIVE_FILE) + + node_page_state(pgdat, NR_INACTIVE_FILE) + + node_page_state(pgdat, NR_ISOLATED_FILE); + dirty = node_page_state(pgdat, NR_FILE_DIRTY); + /* + * node_page_state() sum can go out of sync since + * all the values are not read at once. + */ + if (likely(reclaimable_file > dirty)) + clean = reclaimable_file - dirty; + else + clean = 0; + + sc->clean_below_low = clean < sysctl_clean_low_ratio_kb; + sc->clean_below_min = clean < sysctl_clean_min_ratio_kb; + } else { + sc->clean_below_low = 0; + sc->clean_below_min = 0; + } +} + #ifdef CONFIG_LRU_GEN #ifdef CONFIG_LRU_GEN_ENABLED @@ -4623,13 +4756,20 @@ static int get_tier_idx(struct lruvec *lruvec, int type) return tier - 1; } -static int get_type_to_scan(struct lruvec *lruvec, int swappiness) +static int get_type_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness) { struct ctrl_pos sp, pv; if (swappiness <= MIN_SWAPPINESS + 1) return LRU_GEN_FILE; + if (sc->clean_below_min) + return LRU_GEN_ANON; + if (sc->anon_below_min) + return LRU_GEN_FILE; + if (sc->clean_below_low) + return LRU_GEN_ANON; + if (swappiness >= MAX_SWAPPINESS) return LRU_GEN_ANON; /* @@ -4646,7 +4786,7 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw int *type_scanned, struct list_head *list) { int i; - int type = get_type_to_scan(lruvec, swappiness); + int type = get_type_to_scan(lruvec, sc, swappiness); for_each_evictable_type(i, swappiness) { int scanned; @@ -4889,6 +5029,8 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); + prepare_workingset_protection(pgdat, sc); + /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ if (mem_cgroup_below_min(NULL, memcg)) return MEMCG_LRU_YOUNG; @@ -6027,6 +6169,8 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) prepare_scan_control(pgdat, sc); + prepare_workingset_protection(pgdat, sc); + shrink_node_memcgs(pgdat, sc); flush_reclaim_state(sc); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index dd5cf8914a28..b216088a6abd 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -632,7 +632,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo) * having to remove and re-insert us on the wait queue. */ for (;;) { - prepare_to_wait_exclusive(sk_sleep(sk), &wait, + prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); release_sock(sk); if (reqsk_queue_empty(&icsk->icsk_accept_queue)) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 13dcd86e74ca..338e1aec0eaa 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -50,18 +50,23 @@ endif # =========================================================================== +builtin_suffix := $(if $(filter %.a_thinlto_native, $(MAKECMDGOALS)),.a_thinlto_native,.a) +ifeq ($(thinlto_final_pass),1) +builtin_suffix :=.a_thinlto_native +endif + # subdir-builtin and subdir-modorder may contain duplications. Use $(sort ...) -subdir-builtin := $(sort $(filter %/built-in.a, $(real-obj-y))) +subdir-builtin := $(sort $(filter %/built-in$(builtin_suffix), $(real-obj-y))) subdir-modorder := $(sort $(filter %/modules.order, $(obj-m))) targets-for-builtin := $(extra-y) ifneq ($(strip $(lib-y) $(lib-m) $(lib-)),) -targets-for-builtin += $(obj)/lib.a +targets-for-builtin += $(obj)/lib$(builtin_suffix) endif ifdef need-builtin -targets-for-builtin += $(obj)/built-in.a +targets-for-builtin += $(obj)/built-in$(builtin_suffix) endif targets-for-modules := $(foreach x, o mod, \ @@ -337,6 +342,10 @@ $(obj)/%.o: $(obj)/%.S FORCE targets += $(filter-out $(subdir-builtin), $(real-obj-y)) targets += $(filter-out $(subdir-modorder), $(real-obj-m)) targets += $(lib-y) $(always-y) +ifeq ($(builtin_suffix),.a_thinlto_native) +native_targets = $(patsubst,%.o,%.o_thinlto_native,$(targets)) +targets += $(native_targets) +endif # Linker scripts preprocessor (.lds.S -> .lds) # --------------------------------------------------------------------------- @@ -347,6 +356,24 @@ quiet_cmd_cpp_lds_S = LDS $@ $(obj)/%.lds: $(src)/%.lds.S FORCE $(call if_changed_dep,cpp_lds_S) +ifdef CONFIG_LTO_CLANG_THIN_DIST +# Generate .o_thinlto_native (obj) from .o (bitcode) file +# --------------------------------------------------------------------------- +quiet_cmd_cc_o_bc = CC $(quiet_modtag) $@ + +cmd_cc_o_bc = $(if $(filter bitcode, $(shell file -b $<)),$(CC) \ + $(filter-out -Wp% $(LINUXINCLUDE) %.h.gch %.h -D% \ + -flto=thin, $(c_flags)) \ + -Wno-unused-command-line-argument \ + -x ir -fthinlto-index=$<.thinlto.bc -c -o $@ \ + $(if $(findstring ../,$<), \ + $$(realpath --relative-to=$(srcroot) $<), $<), \ + cp $< $@) + +$(obj)/%.o_thinlto_native: $(obj)/%.o FORCE + $(call if_changed,cc_o_bc) +endif + # ASN.1 grammar # --------------------------------------------------------------------------- quiet_cmd_asn1_compiler = ASN.1 $(basename $@).[ch] @@ -360,7 +387,7 @@ $(obj)/%.asn1.c $(obj)/%.asn1.h: $(src)/%.asn1 $(objtree)/scripts/asn1_compiler # --------------------------------------------------------------------------- # To build objects in subdirs, we need to descend into the directories -$(subdir-builtin): $(obj)/%/built-in.a: $(obj)/% ; +$(subdir-builtin): $(obj)/%/built-in$(builtin_suffix): $(obj)/% ; $(subdir-modorder): $(obj)/%/modules.order: $(obj)/% ; # @@ -377,6 +404,12 @@ quiet_cmd_ar_builtin = AR $@ $(obj)/built-in.a: $(real-obj-y) FORCE $(call if_changed,ar_builtin) +ifdef CONFIG_LTO_CLANG_THIN_DIST +# Rule to compile a set of .o_thinlto_native files into one .a_thinlto_native file. +$(obj)/built-in.a_thinlto_native: $(patsubst %.o,%.o_thinlto_native,$(real-obj-y)) FORCE + $(call if_changed,ar_builtin) +endif + # This is a list of build artifacts from the current Makefile and its # sub-directories. The timestamp should be updated when any of the member files. @@ -394,6 +427,14 @@ $(obj)/modules.order: $(obj-m) FORCE $(obj)/lib.a: $(lib-y) FORCE $(call if_changed,ar) +ifdef CONFIG_LTO_CLANG_THIN_DIST +quiet_cmd_ar_native = AR $@ + cmd_ar_native = rm -f $@; $(AR) cDPrsT $@ $(patsubst %.o,%.o_thinlto_native,$(real-prereqs)) + +$(obj)/lib.a_thinlto_native: $(patsubst %.o,%.o_thinlto_native,$(lib-y)) FORCE + $(call if_changed,ar_native) +endif + quiet_cmd_ld_multi_m = LD [M] $@ cmd_ld_multi_m = $(LD) $(ld_flags) -r -o $@ @$< $(cmd_objtool) @@ -459,7 +500,8 @@ $(single-subdir-goals): $(single-subdirs) PHONY += $(subdir-ym) $(subdir-ym): $(Q)$(MAKE) $(build)=$@ \ - need-builtin=$(if $(filter $@/built-in.a, $(subdir-builtin)),1) \ + need-builtin=$(if $(filter $@/built-in$(builtin_suffix), $(subdir-builtin)),1) \ + thinlto_final_pass=$(if $(filter .a_thinlto_native, $(builtin_suffix)),1) \ need-modorder=$(if $(filter $@/modules.order, $(subdir-modorder)),1) \ $(filter $@/%, $(single-subdir-goals)) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 2fe73cda0bdd..9cfd23590334 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -34,8 +34,13 @@ else obj-m := $(filter-out %/, $(obj-m)) endif +builtin_suffix := $(if $(filter %.a_thinlto_native, $(MAKECMDGOALS)),.a_thinlto_native,.a) +ifeq ($(thinlto_final_pass),1) + builtin_suffix :=.a_thinlto_native +endif + ifdef need-builtin -obj-y := $(patsubst %/, %/built-in.a, $(obj-y)) +obj-y := $(patsubst %/, %/built-in$(builtin_suffix), $(obj-y)) else obj-y := $(filter-out %/, $(obj-y)) endif diff --git a/scripts/Makefile.vmlinux_o b/scripts/Makefile.vmlinux_o index b024ffb3e201..f9abc45a68b3 100644 --- a/scripts/Makefile.vmlinux_o +++ b/scripts/Makefile.vmlinux_o @@ -9,6 +9,14 @@ include $(srctree)/scripts/Kbuild.include # for objtool include $(srctree)/scripts/Makefile.lib +ifeq ($(thinlto_final_pass),1) +vmlinux_a := vmlinux.a_thinlto_native +vmlinux_libs := $(patsubst %.a,%.a_thinlto_native,$(KBUILD_VMLINUX_LIBS)) +else +vmlinux_a := vmlinux.a +vmlinux_libs := $(KBUILD_VMLINUX_LIBS) +endif + # Generate a linker script to ensure correct ordering of initcalls for Clang LTO # --------------------------------------------------------------------------- @@ -18,7 +26,7 @@ quiet_cmd_gen_initcalls_lds = GEN $@ $(PERL) $(real-prereqs) > $@ .tmp_initcalls.lds: $(srctree)/scripts/generate_initcall_order.pl \ - vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE + $(vmlinux_a) $(vmlinux_libs) FORCE $(call if_changed,gen_initcalls_lds) targets := .tmp_initcalls.lds @@ -59,8 +67,8 @@ quiet_cmd_ld_vmlinux.o = LD $@ $(LD) ${KBUILD_LDFLAGS} -r -o $@ \ $(vmlinux-o-ld-args-y) \ $(addprefix -T , $(initcalls-lds)) \ - --whole-archive vmlinux.a --no-whole-archive \ - --start-group $(KBUILD_VMLINUX_LIBS) --end-group \ + --whole-archive $(vmlinux_a) --no-whole-archive \ + --start-group $(vmlinux_libs) --end-group \ $(cmd_objtool) define rule_ld_vmlinux.o @@ -68,7 +76,7 @@ define rule_ld_vmlinux.o $(call cmd,gen_objtooldep) endef -vmlinux.o: $(initcalls-lds) vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE +vmlinux.o: $(initcalls-lds) $(vmlinux_a) $(vmlinux_libs) FORCE $(call if_changed_rule,ld_vmlinux.o) targets += vmlinux.o diff --git a/scripts/Makefile.vmlinux_thinlink b/scripts/Makefile.vmlinux_thinlink new file mode 100644 index 000000000000..13e4026c7d45 --- /dev/null +++ b/scripts/Makefile.vmlinux_thinlink @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-only + +PHONY := __default +__default: vmlinux.thinlink + +include include/config/auto.conf +include $(srctree)/scripts/Kbuild.include + + +# Generate a linker script to ensure correct ordering of initcalls for Clang LTO +# --------------------------------------------------------------------------- + +quiet_cmd_gen_initcalls_lds = GEN $@ + cmd_gen_initcalls_lds = \ + $(PYTHON3) $(srctree)/scripts/jobserver-exec \ + $(PERL) $(real-prereqs) > $@ + +.tmp_initcalls_thinlink.lds: $(srctree)/scripts/generate_initcall_order.pl \ + vmlinux.a FORCE + $(call if_changed,gen_initcalls_lds) + +targets := .tmp_initcalls_thinlink.lds + +initcalls-lds := .tmp_initcalls_thinlink.lds + +quiet_cmd_ld_vmlinux.thinlink = LD $@ + cmd_ld_vmlinux.thinlink = \ + $(AR) t vmlinux.a > .vmlinux_thinlto_bc_files; \ + $(LD) ${KBUILD_LDFLAGS} -r $(addprefix -T , $(initcalls-lds)) \ + --thinlto-index-only @.vmlinux_thinlto_bc_files; \ + touch vmlinux.thinlink + +vmlinux.thinlink: vmlinux.a $(initcalls-lds) FORCE + $(call if_changed,ld_vmlinux.thinlink) + +targets += vmlinux.thinlink + +# Add FORCE to the prerequisites of a target to force it to be always rebuilt. +# --------------------------------------------------------------------------- + +PHONY += FORCE +FORCE: + +# Read all saved command lines and dependencies for the $(targets) we +# may be building above, using $(if_changed{,_dep}). As an +# optimization, we don't need to read them if the target does not +# exist, we will rebuild anyway in that case. + +existing-targets := $(wildcard $(sort $(targets))) + +-include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd) + +.PHONY: $(PHONY) diff --git a/scripts/head-object-list.txt b/scripts/head-object-list.txt index 7274dfc65af6..90710b87a387 100644 --- a/scripts/head-object-list.txt +++ b/scripts/head-object-list.txt @@ -18,6 +18,7 @@ arch/arm/kernel/head.o arch/csky/kernel/head.o arch/hexagon/kernel/head.o arch/loongarch/kernel/head.o +arch/loongarch/kernel/head.o_thinlto_native arch/m68k/68000/head.o arch/m68k/coldfire/head.o arch/m68k/kernel/head.o -- 2.49.0.654.g845c48a16a From 297feeecc37cb1a7fba8bb48802c9f6217888d8c Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:55:52 +0200 Subject: [PATCH 7/8] fixes Signed-off-by: Peter Jung --- Documentation/arch/x86/amd-debugging.rst | 368 ++++++++++++++++++ Documentation/arch/x86/index.rst | 1 + arch/Kconfig | 4 +- arch/x86/include/asm/amd/fch.h | 13 + arch/x86/kernel/cpu/amd.c | 54 +++ drivers/bluetooth/btusb.c | 2 + .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 16 +- drivers/gpu/drm/i915/display/intel_dsb.c | 4 + drivers/hid/hid-quirks.c | 5 +- drivers/i2c/busses/Kconfig | 2 +- drivers/i2c/busses/i2c-piix4.c | 18 +- drivers/platform/x86/amd/pmc/pmc-quirks.c | 3 +- include/linux/hid.h | 2 + scripts/package/PKGBUILD | 5 + 14 files changed, 472 insertions(+), 25 deletions(-) create mode 100644 Documentation/arch/x86/amd-debugging.rst create mode 100644 arch/x86/include/asm/amd/fch.h diff --git a/Documentation/arch/x86/amd-debugging.rst b/Documentation/arch/x86/amd-debugging.rst new file mode 100644 index 000000000000..d92bf59d62c7 --- /dev/null +++ b/Documentation/arch/x86/amd-debugging.rst @@ -0,0 +1,368 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Debugging AMD Zen systems ++++++++++++++++++++++++++ + +Introduction +============ + +This document describes techniques that are useful for debugging issues with +AMD Zen systems. It is intended for use by developers and technical users +to help identify and resolve issues. + +S3 vs s2idle +============ + +On AMD systems, it's not possible to simultaneously support suspend-to-RAM (S3) +and suspend-to-idle (s2idle). To confirm which mode your system supports you +can look at ``cat /sys/power/mem_sleep``. If it shows ``s2idle [deep]`` then +*S3* is supported. If it shows ``[s2idle]`` then *s2idle* is +supported. + +On systems that support *S3*, the firmware will be utilized to put all hardware into +the appropriate low power state. + +On systems that support *s2idle*, the kernel will be responsible for transitioning devices +into the appropriate low power state. When all devices are in the appropriate low +power state, the hardware will transition into a hardware sleep state. + +After a suspend cycle you can tell how much time was spent in a hardware sleep +state by looking at ``cat /sys/power/suspend_stats/last_hw_sleep``. + +This flowchart explains how the AMD s2idle suspend flow works. + +.. kernel-figure:: suspend.svg + +This flowchart explains how the amd s2idle resume flow works. + +.. kernel-figure:: resume.svg + +s2idle debugging tool +===================== + +As there are a lot of places that problems can occur, a debugging tool has been +created at +`amd-debug-tools `_ +that can help test for common problems and offer suggestions. + +If you have an s2idle issue, it's best to start with this and follow instructions +from its findings. If you continue to have an issue, raise a bug with the +report generated from this script to +`drm/amd gitlab `_. + +Spurious s2idle wakeups from an IRQ +=================================== + +Spurious wakeups will generally have an IRQ set to ``/sys/power/pm_wakeup_irq``. +This can be matched to ``/proc/interrupts`` to determine what device woke the system. + +If this isn't enough to debug the problem, then the following sysfs files +can be set to add more verbosity to the wakeup process: :: + + # echo 1 | sudo tee /sys/power/pm_debug_messages + # echo 1 | sudo tee /sys/power/pm_print_times + +After making those changes, the kernel will display messages that can +be traced back to kernel s2idle loop code as well as display any active +GPIO sources while waking up. + +If the wakeup is caused by the ACPI SCI, additional ACPI debugging may be +needed. These commands can enable additional trace data: :: + + # echo enable | sudo tee /sys/module/acpi/parameters/trace_state + # echo 1 | sudo tee /sys/module/acpi/parameters/aml_debug_output + # echo 0x0800000f | sudo tee /sys/module/acpi/parameters/debug_level + # echo 0xffff0000 | sudo tee /sys/module/acpi/parameters/debug_layer + +Spurious s2idle wakeups from a GPIO +=================================== + +If a GPIO is active when waking up the system ideally you would look at the +schematic to determine what device it is associated with. If the schematic +is not available, another tactic is to look at the ACPI _EVT() entry +to determine what device is notified when that GPIO is active. + +For a hypothetical example, say that GPIO 59 woke up the system. You can +look at the SSDT to determine what device is notified when GPIO 59 is active. + +First convert the GPIO number into hex. :: + + $ python3 -c "print(hex(59))" + 0x3b + +Next determine which ACPI table has the ``_EVT`` entry. For example: :: + + $ sudo grep EVT /sys/firmware/acpi/tables/SSDT* + grep: /sys/firmware/acpi/tables/SSDT27: binary file matches + +Decode this table:: + + $ sudo cp /sys/firmware/acpi/tables/SSDT27 . + $ sudo iasl -d SSDT27 + +Then look at the table and find the matching entry for GPIO 0x3b. :: + + Case (0x3B) + { + M000 (0x393B) + M460 (" Notify (\\_SB.PCI0.GP17.XHC1, 0x02)\n", Zero, Zero, Zero, Zero, Zero, Zero) + Notify (\_SB.PCI0.GP17.XHC1, 0x02) // Device Wake + } + +You can see in this case that the device ``\_SB.PCI0.GP17.XHC1`` is notified +when GPIO 59 is active. It's obvious this is an XHCI controller, but to go a +step further you can figure out which XHCI controller it is by matching it to +ACPI.:: + + $ grep "PCI0.GP17.XHC1" /sys/bus/acpi/devices/*/path + /sys/bus/acpi/devices/device:2d/path:\_SB_.PCI0.GP17.XHC1 + /sys/bus/acpi/devices/device:2e/path:\_SB_.PCI0.GP17.XHC1.RHUB + /sys/bus/acpi/devices/device:2f/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT1 + /sys/bus/acpi/devices/device:30/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT1.CAM0 + /sys/bus/acpi/devices/device:31/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT1.CAM1 + /sys/bus/acpi/devices/device:32/path:\_SB_.PCI0.GP17.XHC1.RHUB.PRT2 + /sys/bus/acpi/devices/LNXPOWER:0d/path:\_SB_.PCI0.GP17.XHC1.PWRS + +Here you can see it matches to ``device:2d``. Look at the ``physical_node`` +to determine what PCI device that actually is. :: + + $ ls -l /sys/bus/acpi/devices/device:2d/physical_node + lrwxrwxrwx 1 root root 0 Feb 12 13:22 /sys/bus/acpi/devices/device:2d/physical_node -> ../../../../../pci0000:00/0000:00:08.1/0000:c2:00.4 + +So there you have it: the PCI device associated with this GPIO wakeup was ``0000:c2:00.4``. + +The ``amd_s2idle.py`` script will capture most of these artifacts for you. + +s2idle PM debug messages +======================== + +During the s2idle flow on AMD systems, the ACPI LPS0 driver is responsible +to check all uPEP constraints. Failing uPEP constraints does not prevent +s0i3 entry. This means that if some constraints are not met, it is possible +the kernel may attempt to enter s2idle even if there are some known issues. + +To activate PM debugging, either specify ``pm_debug_messagess`` kernel +command-line option at boot or write to ``/sys/power/pm_debug_messages``. +Unmet constraints will be displayed in the kernel log and can be +viewed by logging tools that process kernel ring buffer like ``dmesg`` or +``journalctl``." + +If the system freezes on entry/exit before these messages are flushed, a +useful debugging tactic is to unbind the ``amd_pmc`` driver to prevent +notification to the platform to start s0i3 entry. This will stop the +system from freezing on entry or exit and let you view all the failed +constraints. :: + + cd /sys/bus/platform/drivers/amd_pmc + ls | grep AMD | sudo tee unbind + +After doing this, run the suspend cycle and look specifically for errors around: :: + + ACPI: LPI: Constraint not met; min power state:%s current power state:%s + +Historical examples of s2idle issues +==================================== + +To help understand the types of issues that can occur and how to debug them, +here are some historical examples of s2idle issues that have been resolved. + +Core offlining +-------------- +An end user had reported that taking a core offline would prevent the system +from properly entering s0i3. This was debugged using internal AMD tools +to capture and display a stream of metrics from the hardware showing what changed +when a core was offlined. It was determined that the hardware didn't get +notification the offline cores were in the deepest state, and so it prevented +CPU from going into the deepest state. The issue was debugged to a missing +command to put cores into C3 upon offline. + +`commit d6b88ce2eb9d2 ("ACPI: processor idle: Allow playing dead in C3 state") `_ + +Corruption after resume +----------------------- +A big problem that occurred with Rembrandt was that there was graphical +corruption after resume. This happened because of a misalignment of PSP +and driver responsibility. The PSP will save and restore DMCUB, but the +driver assumed it needed to reset DMCUB on resume. +This actually was a misalignment for earlier silicon as well, but was not +observed. + +`commit 79d6b9351f086 ("drm/amd/display: Don't reinitialize DMCUB on s0ix resume") `_ + +Back to Back suspends fail +-------------------------- +When using a wakeup source that triggers the IRQ to wakeup, a bug in the +pinctrl-amd driver may capture the wrong state of the IRQ and prevent the +system going back to sleep properly. + +`commit b8c824a869f22 ("pinctrl: amd: Don't save/restore interrupt status and wake status bits") `_ + +Spurious timer based wakeup after 5 minutes +------------------------------------------- +The HPET was being used to program the wakeup source for the system, however +this was causing a spurious wakeup after 5 minutes. The correct alarm to use +was the ACPI alarm. + +`commit 3d762e21d5637 ("rtc: cmos: Use ACPI alarm for non-Intel x86 systems too") `_ + +Disk disappears after resume +---------------------------- +After resuming from s2idle, the NVME disk would disappear. This was due to the +BIOS not specifying the _DSD StorageD3Enable property. This caused the NVME +driver not to put the disk into the expected state at suspend and to fail +on resume. + +`commit e79a10652bbd3 ("ACPI: x86: Force StorageD3Enable on more products") `_ + +Spurious IRQ1 +------------- +A number of Renoir, Lucienne, Cezanne, & Barcelo platforms have a +platform firmware bug where IRQ1 is triggered during s0i3 resume. + +This was fixed in the platform firmware, but a number of systems didn't +receive any more platform firmware updates. + +`commit 8e60615e89321 ("platform/x86/amd: pmc: Disable IRQ1 wakeup for RN/CZN") `_ + +Hardware timeout +---------------- +The hardware performs many actions besides accepting the values from +amd-pmc driver. As the communication path with the hardware is a mailbox, +it's possible that it might not respond quickly enough. +This issue manifested as a failure to suspend: :: + + PM: dpm_run_callback(): acpi_subsys_suspend_noirq+0x0/0x50 returns -110 + amd_pmc AMDI0005:00: PM: failed to suspend noirq: error -110 + +The timing problem was identified by comparing the values of the idle mask. + +`commit 3c3c8e88c8712 ("platform/x86: amd-pmc: Increase the response register timeout") `_ + +Failed to reach hardware sleep state with panel on +-------------------------------------------------- +On some Strix systems certain panels were observed to block the system from +entering a hardware sleep state if the internal panel was on during the sequence. + +Even though the panel got turned off during suspend it exposed a timing problem +where an interrupt caused the display hardware to wake up and block low power +state entry. + +`commit 40b8c14936bd2 ("drm/amd/display: Disable unneeded hpd interrupts during dm_init") `_ + +Runtime power consumption issues +================================ + +Runtime power consumption is influenced by many factors, including but not +limited to the configuration of the PCIe Active State Power Management (ASPM), +the display brightness, the EPP policy of the CPU, and the power management +of the devices. + +ASPM +---- +For the best runtime power consumption, ASPM should be programmed as intended +by the BIOS from the hardware vendor. To accomplish this the Linux kernel +should be compiled with ``CONFIG_PCIEASPM_DEFAULT`` set to ``y`` and the +sysfs file ``/sys/module/pcie_aspm/parameters/policy`` should not be modified. + +Most notably, if L1.2 is not configured properly for any devices, the SoC +will not be able to enter the deepest idle state. + +EPP Policy +---------- +The ``energy_performance_preference`` sysfs file can be used to set a bias +of efficiency or performance for a CPU. This has a direct relationship on +the battery life when more heavily biased towards performance. + + +BIOS debug messages +=================== + +Most OEM machines don't have a serial UART for outputting kernel or BIOS +debug messages. However BIOS debug messages are useful for understanding +both BIOS bugs and bugs with the Linux kernel drivers that call BIOS AML. + +As the BIOS on most OEM AMD systems are based off an AMD reference BIOS, +the infrastructure used for exporting debugging messages is often the same +as AMD reference BIOS. + +Manually Parsing +---------------- +There is generally an ACPI method ``\M460`` that different paths of the AML +will call to emit a message to the BIOS serial log. This method takes +7 arguments, with the first being a string and the rest being optional +integers:: + + Method (M460, 7, Serialized) + +Here is an example of a string that BIOS AML may call out using ``\M460``:: + + M460 (" OEM-ASL-PCIe Address (0x%X)._REG (%d %d) PCSA = %d\n", DADR, Arg0, Arg1, PCSA, Zero, Zero) + +Normally when executed, the ``\M460`` method would populate the additional +arguments into the string. In order to get these messages from the Linux +kernel a hook has been added into ACPICA that can capture the *arguments* +sent to ``\M460`` and print them to the kernel ring buffer. +For example the following message could be emitted into kernel ring buffer:: + + extrace-0174 ex_trace_args : " OEM-ASL-PCIe Address (0x%X)._REG (%d %d) PCSA = %d\n", ec106000, 2, 1, 1, 0, 0 + +In order to get these messages, you need to compile with ``CONFIG_ACPI_DEBUG`` +and then turn on the following ACPICA tracing parameters. +This can be done either on the kernel command line or at runtime: + +* ``acpi.trace_method_name=\M460`` +* ``acpi.trace_state=method`` + +NOTE: These can be very noisy at bootup. If you turn these parameters on +the kernel command, please also consider turning up ``CONFIG_LOG_BUF_SHIFT`` +to a larger size such as 17 to avoid losing early boot messages. + +Tool assisted Parsing +--------------------- +As mentioned above, parsing by hand can be tedious, especially with a lot of +messages. To help with this, a tool has been created at +`amd-debug-tools `_ +to help parse the messages. + +Random reboot issues +==================== + +When a random reboot occurs, the high-level reason for the reboot is stored +in a register that will persist onto the next boot. + +There are 6 classes of reasons for the reboot: + * Software induced + * Power state transition + * Pin induced + * Hardware induced + * Remote reset + * Internal CPU event + +.. csv-table:: + :header: "Bit", "Type", "Reason" + :align: left + + "0", "Pin", "thermal pin BP_THERMTRIP_L was tripped" + "1", "Pin", "power button was pressed for 4 seconds" + "2", "Pin", "shutdown pin was tripped" + "4", "Remote", "remote ASF power off command was received" + "9", "Internal", "internal CPU thermal limit was tripped" + "16", "Pin", "system reset pin BP_SYS_RST_L was tripped" + "17", "Software", "software issued PCI reset" + "18", "Software", "software wrote 0x4 to reset control register 0xCF9" + "19", "Software", "software wrote 0x6 to reset control register 0xCF9" + "20", "Software", "software wrote 0xE to reset control register 0xCF9" + "21", "ACPI-state", "ACPI power state transition occurred" + "22", "Pin", "keyboard reset pin KB_RST_L was tripped" + "23", "Internal", "internal CPU shutdown event occurred" + "24", "Hardware", "system failed to boot before failed boot timer expired" + "25", "Hardware", "hardware watchdog timer expired" + "26", "Remote", "remote ASF reset command was received" + "27", "Internal", "an uncorrected error caused a data fabric sync flood event" + "29", "Internal", "FCH and MP1 failed warm reset handshake" + "30", "Internal", "a parity error occurred" + "31", "Internal", "a software sync flood event occurred" + +This information is read by the kernel at bootup and printed into +the syslog. When a random reboot occurs this message can be helpful +to determine the next component to debug. diff --git a/Documentation/arch/x86/index.rst b/Documentation/arch/x86/index.rst index 8ac64d7de4dc..58a006525ae8 100644 --- a/Documentation/arch/x86/index.rst +++ b/Documentation/arch/x86/index.rst @@ -25,6 +25,7 @@ x86-specific Documentation shstk iommu intel_txt + amd-debugging amd-memory-encryption amd_hsmp tdx diff --git a/arch/Kconfig b/arch/Kconfig index 30dccda07c67..c4c83282db05 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1156,7 +1156,7 @@ config ARCH_MMAP_RND_BITS int "Number of bits to use for ASLR of mmap base address" if EXPERT range ARCH_MMAP_RND_BITS_MIN ARCH_MMAP_RND_BITS_MAX default ARCH_MMAP_RND_BITS_DEFAULT if ARCH_MMAP_RND_BITS_DEFAULT - default ARCH_MMAP_RND_BITS_MIN + default ARCH_MMAP_RND_BITS_MAX depends on HAVE_ARCH_MMAP_RND_BITS help This value can be used to select the number of bits to use to @@ -1190,7 +1190,7 @@ config ARCH_MMAP_RND_COMPAT_BITS int "Number of bits to use for ASLR of mmap base address for compatible applications" if EXPERT range ARCH_MMAP_RND_COMPAT_BITS_MIN ARCH_MMAP_RND_COMPAT_BITS_MAX default ARCH_MMAP_RND_COMPAT_BITS_DEFAULT if ARCH_MMAP_RND_COMPAT_BITS_DEFAULT - default ARCH_MMAP_RND_COMPAT_BITS_MIN + default ARCH_MMAP_RND_COMPAT_BITS_MAX depends on HAVE_ARCH_MMAP_RND_COMPAT_BITS help This value can be used to select the number of bits to use to diff --git a/arch/x86/include/asm/amd/fch.h b/arch/x86/include/asm/amd/fch.h new file mode 100644 index 000000000000..2cf5153edbc2 --- /dev/null +++ b/arch/x86/include/asm/amd/fch.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_AMD_FCH_H_ +#define _ASM_X86_AMD_FCH_H_ + +#define FCH_PM_BASE 0xFED80300 + +/* Register offsets from PM base: */ +#define FCH_PM_DECODEEN 0x00 +#define FCH_PM_DECODEEN_SMBUS0SEL GENMASK(20, 19) +#define FCH_PM_SCRATCH 0x80 +#define FCH_PM_S5_RESET_STATUS 0xC0 + +#endif /* _ASM_X86_AMD_FCH_H_ */ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 4e06baab40bb..f9cbaba2949c 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -1242,3 +1243,56 @@ void amd_check_microcode(void) if (cpu_feature_enabled(X86_FEATURE_ZEN2)) on_each_cpu(zenbleed_check_cpu, NULL, 1); } + +static const char * const s5_reset_reason_txt[] = { + [0] = "thermal pin BP_THERMTRIP_L was tripped", + [1] = "power button was pressed for 4 seconds", + [2] = "shutdown pin was tripped", + [4] = "remote ASF power off command was received", + [9] = "internal CPU thermal limit was tripped", + [16] = "system reset pin BP_SYS_RST_L was tripped", + [17] = "software issued PCI reset", + [18] = "software wrote 0x4 to reset control register 0xCF9", + [19] = "software wrote 0x6 to reset control register 0xCF9", + [20] = "software wrote 0xE to reset control register 0xCF9", + [21] = "ACPI power state transition occurred", + [22] = "keyboard reset pin KB_RST_L was tripped", + [23] = "internal CPU shutdown event occurred", + [24] = "system failed to boot before failed boot timer expired", + [25] = "hardware watchdog timer expired", + [26] = "remote ASF reset command was received", + [27] = "an uncorrected error caused a data fabric sync flood event", + [29] = "FCH and MP1 failed warm reset handshake", + [30] = "a parity error occurred", + [31] = "a software sync flood event occurred", +}; + +static __init int print_s5_reset_status_mmio(void) +{ + unsigned long value; + void __iomem *addr; + int i; + + if (!cpu_feature_enabled(X86_FEATURE_ZEN)) + return 0; + + addr = ioremap(FCH_PM_BASE + FCH_PM_S5_RESET_STATUS, sizeof(value)); + if (!addr) + return 0; + + value = ioread32(addr); + iounmap(addr); + + for (i = 0; i < ARRAY_SIZE(s5_reset_reason_txt); i++) { + if (!(value & BIT(i))) + continue; + + if (s5_reset_reason_txt[i]) { + pr_info("x86/amd: Previous system reset reason [0x%08lx]: %s\n", + value, s5_reset_reason_txt[i]); + } + } + + return 0; +} +late_initcall(print_s5_reset_status_mmio); diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 256b451bbe06..bbbebe3a2f56 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -702,6 +702,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe139), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0489, 0xe14e), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe14f), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe150), .driver_info = BTUSB_MEDIATEK | diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index ef0f71ce25d5..37b14f31cc46 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -675,21 +675,15 @@ static void dm_crtc_high_irq(void *interrupt_params) spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); if (acrtc->dm_irq_params.stream && - acrtc->dm_irq_params.vrr_params.supported) { - bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; - bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; - bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; - + acrtc->dm_irq_params.vrr_params.supported && + acrtc->dm_irq_params.freesync_config.state == + VRR_STATE_ACTIVE_VARIABLE) { mod_freesync_handle_v_update(adev->dm.freesync_module, acrtc->dm_irq_params.stream, &acrtc->dm_irq_params.vrr_params); - /* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */ - if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { - dc_stream_adjust_vmin_vmax(adev->dm.dc, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - } + dc_stream_adjust_vmin_vmax(adev->dm.dc, acrtc->dm_irq_params.stream, + &acrtc->dm_irq_params.vrr_params.adjust); } /* diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c index 9fc4003d1579..e5ece83a1df1 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.c +++ b/drivers/gpu/drm/i915/display/intel_dsb.c @@ -806,6 +806,10 @@ struct intel_dsb *intel_dsb_prepare(struct intel_atomic_state *state, if (!i915->display.params.enable_dsb) return NULL; + /* TODO: DSB is broken in Xe KMD, so disabling it until fixed */ + if (!IS_ENABLED(I915)) + return NULL; + dsb = kzalloc(sizeof(*dsb), GFP_KERNEL); if (!dsb) goto out; diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 0731473cc9b1..7fefeb413ec3 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -1063,7 +1063,7 @@ bool hid_ignore(struct hid_device *hdev) } if (hdev->type == HID_TYPE_USBMOUSE && - hid_match_id(hdev, hid_mouse_ignore_list)) + hdev->quirks & HID_QUIRK_IGNORE_MOUSE) return true; return !!hid_match_id(hdev, hid_ignore_list); @@ -1267,6 +1267,9 @@ static unsigned long hid_gets_squirk(const struct hid_device *hdev) if (hid_match_id(hdev, hid_ignore_list)) quirks |= HID_QUIRK_IGNORE; + if (hid_match_id(hdev, hid_mouse_ignore_list)) + quirks |= HID_QUIRK_IGNORE_MOUSE; + if (hid_match_id(hdev, hid_have_special_driver)) quirks |= HID_QUIRK_HAVE_SPECIAL_DRIVER; diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 83c88c79afe2..bbbd6240fa6e 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -200,7 +200,7 @@ config I2C_ISMT config I2C_PIIX4 tristate "Intel PIIX4 and compatible (ATI/AMD/Serverworks/Broadcom/SMSC)" - depends on PCI && HAS_IOPORT + depends on PCI && HAS_IOPORT && X86 select I2C_SMBUS help If you say yes to this option, support will be included for the Intel diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c index dd75916157f0..59ecaa990bce 100644 --- a/drivers/i2c/busses/i2c-piix4.c +++ b/drivers/i2c/busses/i2c-piix4.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "i2c-piix4.h" @@ -80,12 +81,11 @@ #define SB800_PIIX4_PORT_IDX_MASK 0x06 #define SB800_PIIX4_PORT_IDX_SHIFT 1 -/* On kerncz and Hudson2, SmBus0Sel is at bit 20:19 of PMx00 DecodeEn */ -#define SB800_PIIX4_PORT_IDX_KERNCZ 0x02 -#define SB800_PIIX4_PORT_IDX_MASK_KERNCZ 0x18 +/* SmBus0Sel is at bit 20:19 of PMx00 DecodeEn */ +#define SB800_PIIX4_PORT_IDX_KERNCZ (FCH_PM_DECODEEN + 0x02) +#define SB800_PIIX4_PORT_IDX_MASK_KERNCZ (FCH_PM_DECODEEN_SMBUS0SEL >> 16) #define SB800_PIIX4_PORT_IDX_SHIFT_KERNCZ 3 -#define SB800_PIIX4_FCH_PM_ADDR 0xFED80300 #define SB800_PIIX4_FCH_PM_SIZE 8 #define SB800_ASF_ACPI_PATH "\\_SB.ASFC" @@ -162,19 +162,19 @@ int piix4_sb800_region_request(struct device *dev, struct sb800_mmio_cfg *mmio_c if (mmio_cfg->use_mmio) { void __iomem *addr; - if (!request_mem_region_muxed(SB800_PIIX4_FCH_PM_ADDR, + if (!request_mem_region_muxed(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE, "sb800_piix4_smb")) { dev_err(dev, "SMBus base address memory region 0x%x already in use.\n", - SB800_PIIX4_FCH_PM_ADDR); + FCH_PM_BASE); return -EBUSY; } - addr = ioremap(SB800_PIIX4_FCH_PM_ADDR, + addr = ioremap(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE); if (!addr) { - release_mem_region(SB800_PIIX4_FCH_PM_ADDR, + release_mem_region(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE); dev_err(dev, "SMBus base address mapping failed.\n"); return -ENOMEM; @@ -201,7 +201,7 @@ void piix4_sb800_region_release(struct device *dev, struct sb800_mmio_cfg *mmio_ { if (mmio_cfg->use_mmio) { iounmap(mmio_cfg->addr); - release_mem_region(SB800_PIIX4_FCH_PM_ADDR, + release_mem_region(FCH_PM_BASE, SB800_PIIX4_FCH_PM_SIZE); return; } diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c index 2e3f6fc67c56..5c7c01f66cde 100644 --- a/drivers/platform/x86/amd/pmc/pmc-quirks.c +++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "pmc.h" @@ -20,7 +21,7 @@ struct quirk_entry { }; static struct quirk_entry quirk_s2idle_bug = { - .s2idle_bug_mmio = 0xfed80380, + .s2idle_bug_mmio = FCH_PM_BASE + FCH_PM_SCRATCH, }; static struct quirk_entry quirk_spurious_8042 = { diff --git a/include/linux/hid.h b/include/linux/hid.h index ef9a90ca0fbd..234828b8ae82 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -357,6 +357,7 @@ struct hid_item { * | @HID_QUIRK_INPUT_PER_APP: * | @HID_QUIRK_X_INVERT: * | @HID_QUIRK_Y_INVERT: + * | @HID_QUIRK_IGNORE_MOUSE: * | @HID_QUIRK_SKIP_OUTPUT_REPORTS: * | @HID_QUIRK_SKIP_OUTPUT_REPORT_ID: * | @HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP: @@ -382,6 +383,7 @@ struct hid_item { #define HID_QUIRK_INPUT_PER_APP BIT(11) #define HID_QUIRK_X_INVERT BIT(12) #define HID_QUIRK_Y_INVERT BIT(13) +#define HID_QUIRK_IGNORE_MOUSE BIT(14) #define HID_QUIRK_SKIP_OUTPUT_REPORTS BIT(16) #define HID_QUIRK_SKIP_OUTPUT_REPORT_ID BIT(17) #define HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP BIT(18) diff --git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD index 452374d63c24..08f80d7c5df0 100644 --- a/scripts/package/PKGBUILD +++ b/scripts/package/PKGBUILD @@ -90,6 +90,11 @@ _package-headers() { "${srctree}/scripts/package/install-extmod-build" "${builddir}" fi + # required when DEBUG_INFO_BTF_MODULES is enabled + if [ -f tools/bpf/resolve_btfids/resolve_btfids ]; then + install -Dt "$builddir/tools/bpf/resolve_btfids" tools/bpf/resolve_btfids/resolve_btfids + fi + echo "Installing System.map and config..." mkdir -p "${builddir}" cp System.map "${builddir}/System.map" -- 2.49.0.654.g845c48a16a From 4cd7518f514ab1b1e1355b69f4ed0cfd6064b027 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 26 May 2025 14:56:21 +0200 Subject: [PATCH 8/8] t2 Signed-off-by: Peter Jung --- Documentation/core-api/printk-formats.rst | 32 + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 + drivers/gpu/drm/i915/display/intel_ddi.c | 4 + drivers/gpu/drm/i915/display/intel_fbdev.c | 6 +- drivers/gpu/drm/i915/display/intel_quirks.c | 15 + drivers/gpu/drm/i915/display/intel_quirks.h | 1 + drivers/gpu/drm/tiny/appletbdrm.c | 4 +- drivers/gpu/vga/vga_switcheroo.c | 7 +- drivers/hid/Kconfig | 11 +- drivers/hid/Makefile | 4 + drivers/hid/dockchannel-hid/Kconfig | 14 + drivers/hid/dockchannel-hid/Makefile | 6 + drivers/hid/dockchannel-hid/dockchannel-hid.c | 1213 +++++++++++++++++ drivers/hid/hid-apple.c | 45 +- drivers/hid/hid-core.c | 11 +- drivers/hid/hid-ids.h | 25 +- drivers/hid/hid-magicmouse.c | 908 ++++++++++-- drivers/hid/hid-multitouch.c | 60 +- drivers/hid/hid-quirks.c | 9 +- drivers/hid/spi-hid/Kconfig | 26 + drivers/hid/spi-hid/Makefile | 10 + drivers/hid/spi-hid/spi-hid-apple-core.c | 1194 ++++++++++++++++ drivers/hid/spi-hid/spi-hid-apple-of.c | 151 ++ drivers/hid/spi-hid/spi-hid-apple.h | 35 + drivers/hwmon/applesmc.c | 1138 ++++++++++++---- drivers/pci/vgaarb.c | 1 + drivers/platform/x86/apple-gmux.c | 18 + drivers/soc/apple/Kconfig | 24 + drivers/soc/apple/Makefile | 6 + drivers/soc/apple/dockchannel.c | 406 ++++++ drivers/soc/apple/rtkit-helper.c | 151 ++ drivers/staging/Kconfig | 2 + drivers/staging/Makefile | 1 + drivers/staging/apple-bce/Kconfig | 18 + drivers/staging/apple-bce/Makefile | 28 + drivers/staging/apple-bce/apple_bce.c | 445 ++++++ drivers/staging/apple-bce/apple_bce.h | 38 + drivers/staging/apple-bce/audio/audio.c | 711 ++++++++++ drivers/staging/apple-bce/audio/audio.h | 125 ++ drivers/staging/apple-bce/audio/description.h | 42 + drivers/staging/apple-bce/audio/pcm.c | 308 +++++ drivers/staging/apple-bce/audio/pcm.h | 16 + drivers/staging/apple-bce/audio/protocol.c | 347 +++++ drivers/staging/apple-bce/audio/protocol.h | 147 ++ .../staging/apple-bce/audio/protocol_bce.c | 226 +++ .../staging/apple-bce/audio/protocol_bce.h | 72 + drivers/staging/apple-bce/mailbox.c | 151 ++ drivers/staging/apple-bce/mailbox.h | 53 + drivers/staging/apple-bce/queue.c | 390 ++++++ drivers/staging/apple-bce/queue.h | 177 +++ drivers/staging/apple-bce/queue_dma.c | 220 +++ drivers/staging/apple-bce/queue_dma.h | 50 + drivers/staging/apple-bce/vhci/command.h | 204 +++ drivers/staging/apple-bce/vhci/queue.c | 268 ++++ drivers/staging/apple-bce/vhci/queue.h | 76 ++ drivers/staging/apple-bce/vhci/transfer.c | 661 +++++++++ drivers/staging/apple-bce/vhci/transfer.h | 73 + drivers/staging/apple-bce/vhci/vhci.c | 759 +++++++++++ drivers/staging/apple-bce/vhci/vhci.h | 52 + include/linux/hid.h | 6 +- include/linux/soc/apple/dockchannel.h | 26 + lib/tests/printf_kunit.c | 39 +- lib/vsprintf.c | 40 +- scripts/checkpatch.pl | 2 +- 64 files changed, 10856 insertions(+), 455 deletions(-) create mode 100644 drivers/hid/dockchannel-hid/Kconfig create mode 100644 drivers/hid/dockchannel-hid/Makefile create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid.c create mode 100644 drivers/hid/spi-hid/Kconfig create mode 100644 drivers/hid/spi-hid/Makefile create mode 100644 drivers/hid/spi-hid/spi-hid-apple-core.c create mode 100644 drivers/hid/spi-hid/spi-hid-apple-of.c create mode 100644 drivers/hid/spi-hid/spi-hid-apple.h create mode 100644 drivers/soc/apple/dockchannel.c create mode 100644 drivers/soc/apple/rtkit-helper.c create mode 100644 drivers/staging/apple-bce/Kconfig create mode 100644 drivers/staging/apple-bce/Makefile create mode 100644 drivers/staging/apple-bce/apple_bce.c create mode 100644 drivers/staging/apple-bce/apple_bce.h create mode 100644 drivers/staging/apple-bce/audio/audio.c create mode 100644 drivers/staging/apple-bce/audio/audio.h create mode 100644 drivers/staging/apple-bce/audio/description.h create mode 100644 drivers/staging/apple-bce/audio/pcm.c create mode 100644 drivers/staging/apple-bce/audio/pcm.h create mode 100644 drivers/staging/apple-bce/audio/protocol.c create mode 100644 drivers/staging/apple-bce/audio/protocol.h create mode 100644 drivers/staging/apple-bce/audio/protocol_bce.c create mode 100644 drivers/staging/apple-bce/audio/protocol_bce.h create mode 100644 drivers/staging/apple-bce/mailbox.c create mode 100644 drivers/staging/apple-bce/mailbox.h create mode 100644 drivers/staging/apple-bce/queue.c create mode 100644 drivers/staging/apple-bce/queue.h create mode 100644 drivers/staging/apple-bce/queue_dma.c create mode 100644 drivers/staging/apple-bce/queue_dma.h create mode 100644 drivers/staging/apple-bce/vhci/command.h create mode 100644 drivers/staging/apple-bce/vhci/queue.c create mode 100644 drivers/staging/apple-bce/vhci/queue.h create mode 100644 drivers/staging/apple-bce/vhci/transfer.c create mode 100644 drivers/staging/apple-bce/vhci/transfer.h create mode 100644 drivers/staging/apple-bce/vhci/vhci.c create mode 100644 drivers/staging/apple-bce/vhci/vhci.h create mode 100644 include/linux/soc/apple/dockchannel.h diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst index 4bdc394e86af..f531873bb3c9 100644 --- a/Documentation/core-api/printk-formats.rst +++ b/Documentation/core-api/printk-formats.rst @@ -648,6 +648,38 @@ Examples:: %p4cc Y10 little-endian (0x20303159) %p4cc NV12 big-endian (0xb231564e) +Generic FourCC code +------------------- + +:: + %p4c[h[R]lb] gP00 (0x67503030) + +Print a generic FourCC code, as both ASCII characters and its numerical +value as hexadecimal. + +The generic FourCC code is always printed in the big-endian format, +the most significant byte first. This is the opposite of V4L/DRM FourCCs. + +The additional ``h``, ``hR``, ``l``, and ``b`` specifiers define what +endianness is used to load the stored bytes. The data might be interpreted +using the host, reversed host byte order, little-endian, or big-endian. + +Passed by reference. + +Examples for a little-endian machine, given &(u32)0x67503030:: + + %p4ch gP00 (0x67503030) + %p4chR 00Pg (0x30305067) + %p4cl gP00 (0x67503030) + %p4cb 00Pg (0x30305067) + +Examples for a big-endian machine, given &(u32)0x67503030:: + + %p4ch gP00 (0x67503030) + %p4chR 00Pg (0x30305067) + %p4cl 00Pg (0x30305067) + %p4cb gP00 (0x67503030) + Rust ---- diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 6ef3fedc1233..75bdd8b8b6bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2287,6 +2287,9 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, return -EINVAL; } + if (vga_switcheroo_client_probe_defer(pdev)) + return -EPROBE_DEFER; + /* skip devices which are owned by radeon */ for (i = 0; i < ARRAY_SIZE(amdgpu_unsupported_pciidlist); i++) { if (amdgpu_unsupported_pciidlist[i] == pdev->device) diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index f38c998935b9..1df3ddb366f4 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -4848,6 +4848,7 @@ static int intel_ddi_init_hdmi_connector(struct intel_digital_port *dig_port) static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dig_port) { + struct intel_display *display = to_intel_display(dig_port); struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); if (dig_port->base.port != PORT_A) @@ -4856,6 +4857,9 @@ static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dig_port) if (dig_port->ddi_a_4_lanes) return false; + if (intel_has_quirk(display, QUIRK_DDI_A_FORCE_4_LANES)) + return true; + /* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only * supported configuration */ diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index adc19d5607de..64882765aa0a 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -224,10 +224,10 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, ifbdev->fb = NULL; if (fb && - (sizes->fb_width > fb->base.width || - sizes->fb_height > fb->base.height)) { + (sizes->fb_width != fb->base.width || + sizes->fb_height != fb->base.height)) { drm_dbg_kms(&dev_priv->drm, - "BIOS fb too small (%dx%d), we require (%dx%d)," + "BIOS fb not valid (%dx%d), we require (%dx%d)," " releasing it\n", fb->base.width, fb->base.height, sizes->fb_width, sizes->fb_height); diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index a32fae510ed2..662465733ddb 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -66,6 +66,18 @@ static void quirk_increase_ddi_disabled_time(struct intel_display *display) drm_info(display->drm, "Applying Increase DDI Disabled quirk\n"); } +/* + * In some cases, the firmware might not set the lane count to 4 (for example, + * when booting in some dual GPU Macs with the dGPU as the default GPU), this + * quirk is used to force it as otherwise it might not be possible to compute a + * valid link configuration. + */ +static void quirk_ddi_a_force_4_lanes(struct intel_display *display) +{ + intel_set_quirk(display, QUIRK_DDI_A_FORCE_4_LANES); + drm_info(display->drm, "Applying DDI A Forced 4 Lanes quirk\n"); +} + static void quirk_no_pps_backlight_power_hook(struct intel_display *display) { intel_set_quirk(display, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK); @@ -231,6 +243,9 @@ static struct intel_quirk intel_quirks[] = { { 0x3184, 0x1019, 0xa94d, quirk_increase_ddi_disabled_time }, /* HP Notebook - 14-r206nv */ { 0x0f31, 0x103c, 0x220f, quirk_invert_brightness }, + + /* Apple MacBookPro15,1 */ + { 0x3e9b, 0x106b, 0x0176, quirk_ddi_a_force_4_lanes }, }; static const struct intel_dpcd_quirk intel_dpcd_quirks[] = { diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h index cafdebda7535..a5296f82776e 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.h +++ b/drivers/gpu/drm/i915/display/intel_quirks.h @@ -20,6 +20,7 @@ enum intel_quirk_id { QUIRK_LVDS_SSC_DISABLE, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK, QUIRK_FW_SYNC_LEN, + QUIRK_DDI_A_FORCE_4_LANES, }; void intel_init_quirks(struct intel_display *display); diff --git a/drivers/gpu/drm/tiny/appletbdrm.c b/drivers/gpu/drm/tiny/appletbdrm.c index 4370ba22dd88..8643216bad98 100644 --- a/drivers/gpu/drm/tiny/appletbdrm.c +++ b/drivers/gpu/drm/tiny/appletbdrm.c @@ -214,7 +214,7 @@ static int appletbdrm_read_response(struct appletbdrm_device *adev, } if (response->msg != expected_response) { - drm_err(drm, "Unexpected response from device (expected %p4cc found %p4cc)\n", + drm_err(drm, "Unexpected response from device (expected %p4cl found %p4cl)\n", &expected_response, &response->msg); return -EIO; } @@ -288,7 +288,7 @@ static int appletbdrm_get_information(struct appletbdrm_device *adev) } if (pixel_format != APPLETBDRM_PIXEL_FORMAT) { - drm_err(drm, "Encountered unknown pixel format (%p4cc)\n", &pixel_format); + drm_err(drm, "Encountered unknown pixel format (%p4cl)\n", &pixel_format); ret = -EINVAL; goto free_info; } diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c index 18f2c92beff8..3de1bca45ed2 100644 --- a/drivers/gpu/vga/vga_switcheroo.c +++ b/drivers/gpu/vga/vga_switcheroo.c @@ -438,12 +438,7 @@ find_active_client(struct list_head *head) bool vga_switcheroo_client_probe_defer(struct pci_dev *pdev) { if ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY) { - /* - * apple-gmux is needed on pre-retina MacBook Pro - * to probe the panel if pdev is the inactive GPU. - */ - if (apple_gmux_present() && pdev != vga_default_device() && - !vgasr_priv.handler_flags) + if (apple_gmux_present() && !vgasr_priv.handler_flags) return true; } diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index f83b415092ba..56c1c86a2c1e 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -129,7 +129,7 @@ config HID_APPLE tristate "Apple {i,Power,Mac}Books" depends on LEDS_CLASS depends on NEW_LEDS - default !EXPERT + default !EXPERT || SPI_HID_APPLE help Support for some Apple devices which less or more break HID specification. @@ -715,11 +715,13 @@ config LOGIWHEELS_FF config HID_MAGICMOUSE tristate "Apple Magic Mouse/Trackpad multi-touch support" + default SPI_HID_APPLE help Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" @@ -769,6 +771,7 @@ config HID_MULTITOUCH Say Y here if you have one of the following devices: - 3M PCT touch screens - ActionStar dual touch panels + - Touch Bars on x86 MacBook Pros - Atmel panels - Cando dual touch panels - Chunghwa panels @@ -1435,4 +1438,8 @@ endif # HID source "drivers/hid/usbhid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 85af5331ebd2..a0f58475434c 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -177,3 +177,7 @@ obj-$(CONFIG_ASUS_ALLY_HID) += asus-ally-hid/ obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig new file mode 100644 index 000000000000..8a81d551a83d --- /dev/null +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + default ARCH_APPLE + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile new file mode 100644 index 000000000000..7dba766b047f --- /dev/null +++ b/drivers/hid/dockchannel-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c new file mode 100644 index 000000000000..a712a724ded3 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -0,0 +1,1213 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, HID_FEATURE_REPORT, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%d > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case HID_INPUT_REPORT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%d > %ld)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static void dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index ed34f5cd5a91..ed8b87f7556b 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -486,6 +486,7 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, table = apple2021_fn_keys; else if (hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 || hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 || + hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT || hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213) table = macbookpro_no_esc_fn_keys; else if (hid->product == USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K || @@ -498,6 +499,16 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else if (hid->product >= USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI && hid->product <= USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) table = macbookair_fn_keys; + else if (hid->bus == BUS_HOST || hid->bus == BUS_SPI) + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = macbookpro_dedicated_esc_fn_keys; + break; + default: + table = apple2021_fn_keys; + break; + } else if (hid->product < 0x21d || hid->product >= 0x300) table = powerbook_fn_keys; else @@ -910,6 +921,13 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + + if (quirks & APPLE_IGNORE_MOUSE && hdev->type == HID_TYPE_USBMOUSE) + return -ENODEV; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -1127,21 +1145,28 @@ static const struct hid_device_id apple_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS), .driver_data = APPLE_HAS_FN | APPLE_RDESC_JIS }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT), + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), - .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_BACKLIGHT_CTL | APPLE_ISO_TILDE_QUIRK | + APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), - .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_IGNORE_MOUSE }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ANSI), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ISO), @@ -1169,6 +1194,10 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT), .driver_data = APPLE_MAGIC_BACKLIGHT }, diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 4741ff626771..b991ed962366 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -464,7 +464,10 @@ static int hid_parser_global(struct hid_parser *parser, struct hid_item *item) case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; @@ -2294,6 +2297,12 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_I2C: bus = "I2C"; break; + case BUS_SPI: + bus = "SPI"; + break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 594cde034707..70efd28b6bc3 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -93,6 +93,8 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -172,14 +174,15 @@ #define USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI 0x0272 #define USB_DEVICE_ID_APPLE_WELLSPRING9_ISO 0x0273 #define USB_DEVICE_ID_APPLE_WELLSPRING9_JIS 0x0274 -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K 0x027a -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 0x027b -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 0x027c -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213 0x027d -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K 0x027e -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223 0x027f -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K 0x0280 -#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F 0x0340 +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K 0x027a +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132 0x027b +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680 0x027c +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT 0x0278 +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213 0x027d +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K 0x027e +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223 0x027f +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K 0x0280 +#define USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F 0x0340 #define USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY 0x030a #define USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY 0x030b #define USB_DEVICE_ID_APPLE_IRCONTROL 0x8240 @@ -193,6 +196,12 @@ #define USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021 0x029f #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASETEK 0x2433 #define USB_DEVICE_ID_ASETEK_INVICTA 0xf300 diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index adfa329e917b..0787581a25cc 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -60,8 +60,14 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 +#define MTP_REPORT_ID 0x75 +#define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_MS 60000 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -112,30 +118,156 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +#define J140K_TP_DIMENSION_X (float)12100 +#define J140K_TP_MIN_X -5318 +#define J140K_TP_MAX_X 5787 +#define J140K_TP_RES_X \ + ((J140K_TP_MAX_X - J140K_TP_MIN_X) / (J140K_TP_DIMENSION_X / 100)) +#define J140K_TP_DIMENSION_Y (float)8200 +#define J140K_TP_MIN_Y -157 +#define J140K_TP_MAX_Y 7102 +#define J140K_TP_RES_Y \ + ((J140K_TP_MAX_Y - J140K_TP_MIN_Y) / (J140K_TP_DIMENSION_Y / 100)) + +#define J132_TP_DIMENSION_X (float)13500 +#define J132_TP_MIN_X -6243 +#define J132_TP_MAX_X 6749 +#define J132_TP_RES_X \ + ((J132_TP_MAX_X - J132_TP_MIN_X) / (J132_TP_DIMENSION_X / 100)) +#define J132_TP_DIMENSION_Y (float)8400 +#define J132_TP_MIN_Y -170 +#define J132_TP_MAX_Y 7685 +#define J132_TP_RES_Y \ + ((J132_TP_MAX_Y - J132_TP_MIN_Y) / (J132_TP_DIMENSION_Y / 100)) + +#define J680_TP_DIMENSION_X (float)16000 +#define J680_TP_MIN_X -7456 +#define J680_TP_MAX_X 7976 +#define J680_TP_RES_X \ + ((J680_TP_MAX_X - J680_TP_MIN_X) / (J680_TP_DIMENSION_X / 100)) +#define J680_TP_DIMENSION_Y (float)10000 +#define J680_TP_MIN_Y -163 +#define J680_TP_MAX_Y 9283 +#define J680_TP_RES_Y \ + ((J680_TP_MAX_Y - J680_TP_MIN_Y) / (J680_TP_DIMENSION_Y / 100)) + +#define J680_ALT_TP_DIMENSION_X (float)16000 +#define J680_ALT_TP_MIN_X -7456 +#define J680_ALT_TP_MAX_X 7976 +#define J680_ALT_TP_RES_X \ + ((J680_ALT_TP_MAX_X - J680_ALT_TP_MIN_X) / (J680_ALT_TP_DIMENSION_X / 100)) +#define J680_ALT_TP_DIMENSION_Y (float)10000 +#define J680_ALT_TP_MIN_Y -163 +#define J680_ALT_TP_MAX_Y 9283 +#define J680_ALT_TP_RES_Y \ + ((J680_ALT_TP_MAX_Y - J680_ALT_TP_MIN_Y) / (J680_ALT_TP_DIMENSION_Y / 100)) + +#define J213_TP_DIMENSION_X (float)13500 +#define J213_TP_MIN_X -6243 +#define J213_TP_MAX_X 6749 +#define J213_TP_RES_X \ + ((J213_TP_MAX_X - J213_TP_MIN_X) / (J213_TP_DIMENSION_X / 100)) +#define J213_TP_DIMENSION_Y (float)8400 +#define J213_TP_MIN_Y -170 +#define J213_TP_MAX_Y 7685 +#define J213_TP_RES_Y \ + ((J213_TP_MAX_Y - J213_TP_MIN_Y) / (J213_TP_DIMENSION_Y / 100)) + +#define J214K_TP_DIMENSION_X (float)13200 +#define J214K_TP_MIN_X -6046 +#define J214K_TP_MAX_X 6536 +#define J214K_TP_RES_X \ + ((J214K_TP_MAX_X - J214K_TP_MIN_X) / (J214K_TP_DIMENSION_X / 100)) +#define J214K_TP_DIMENSION_Y (float)8200 +#define J214K_TP_MIN_Y -164 +#define J214K_TP_MAX_Y 7439 +#define J214K_TP_RES_Y \ + ((J214K_TP_MAX_Y - J214K_TP_MIN_Y) / (J214K_TP_DIMENSION_Y / 100)) + +#define J223_TP_DIMENSION_X (float)13200 +#define J223_TP_MIN_X -6046 +#define J223_TP_MAX_X 6536 +#define J223_TP_RES_X \ + ((J223_TP_MAX_X - J223_TP_MIN_X) / (J223_TP_DIMENSION_X / 100)) +#define J223_TP_DIMENSION_Y (float)8200 +#define J223_TP_MIN_Y -164 +#define J223_TP_MAX_Y 7439 +#define J223_TP_RES_Y \ + ((J223_TP_MAX_Y - J223_TP_MIN_Y) / (J223_TP_DIMENSION_Y / 100)) + +#define J230K_TP_DIMENSION_X (float)12100 +#define J230K_TP_MIN_X -5318 +#define J230K_TP_MAX_X 5787 +#define J230K_TP_RES_X \ + ((J230K_TP_MAX_X - J230K_TP_MIN_X) / (J230K_TP_DIMENSION_X / 100)) +#define J230K_TP_DIMENSION_Y (float)8200 +#define J230K_TP_MIN_Y -157 +#define J230K_TP_MAX_Y 7102 +#define J230K_TP_RES_Y \ + ((J230K_TP_MAX_Y - J230K_TP_MIN_Y) / (J230K_TP_DIMENSION_Y / 100)) + +#define J152F_TP_DIMENSION_X (float)16000 +#define J152F_TP_MIN_X -7456 +#define J152F_TP_MAX_X 7976 +#define J152F_TP_RES_X \ + ((J152F_TP_MAX_X - J152F_TP_MIN_X) / (J152F_TP_DIMENSION_X / 100)) +#define J152F_TP_DIMENSION_Y (float)10000 +#define J152F_TP_MIN_Y -163 +#define J152F_TP_MAX_Y 9283 +#define J152F_TP_RES_Y \ + ((J152F_TP_MAX_Y - J152F_TP_MIN_Y) / (J152F_TP_DIMENSION_Y / 100)) + +/* These are fallback values, since the real values will be queried from the device. */ +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define T2_TOUCHPAD_ENTRY(model) \ + { USB_DEVICE_ID_APPLE_WELLSPRINGT2_##model, model##_TP_MIN_X, model##_TP_MIN_Y, \ +model##_TP_MAX_X, model##_TP_MAX_Y, model##_TP_RES_X, model##_TP_RES_Y } + +#define INTERNAL_TP_MAX_FINGER_ORIENTATION 16384 + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. * @quirks: Currently unused. + * @query_dimensions: Whether to query and update dimensions on first open * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. + * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. + * @input_ops: Input ops based on device type. */ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; + bool query_dimensions; int ntouches; int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -143,14 +275,186 @@ struct magicmouse_sc { u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static int magicmouse_enable_multitouch(struct hid_device *hdev) +{ + const u8 *feature; + const u8 feature_mt[] = { 0xD7, 0x01 }; + const u8 feature_mt_mouse2[] = { 0xF1, 0x02, 0x01 }; + const u8 feature_mt_trackpad2_usb[] = { 0x02, 0x01 }; + const u8 feature_mt_trackpad2_bt[] = { 0xF1, 0x02, 0x01 }; + u8 *buf; + int ret; + int feature_size; + + switch (hdev->bus) { + case BUS_SPI: + case BUS_HOST: + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + break; + default: + switch (hdev->product) { + case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: + case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: + switch (hdev->vendor) { + case BT_VENDOR_ID_APPLE: + feature_size = sizeof(feature_mt_trackpad2_bt); + feature = feature_mt_trackpad2_bt; + break; + default: /* USB_VENDOR_ID_APPLE */ + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + } + break; + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + break; + case USB_DEVICE_ID_APPLE_MAGICMOUSE2: + feature_size = sizeof(feature_mt_mouse2); + feature = feature_mt_mouse2; + break; + default: + feature_size = sizeof(feature_mt); + feature = feature_mt; + } + } + + buf = kmemdup(feature, feature_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = hid_hw_raw_request(hdev, buf[0], buf, feature_size, + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + kfree(buf); + return ret; +} + +static void magicmouse_enable_mt_work(struct work_struct *work) +{ + struct magicmouse_sc *msc = + container_of(work, struct magicmouse_sc, work.work); + int ret; + + ret = magicmouse_enable_multitouch(msc->hdev); + if (ret < 0) + hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + * + * MTP devices do not need this. + */ + if (hdev->bus != BUS_HOST) { + ret = magicmouse_enable_multitouch(hdev); + if (ret == -EIO && hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + return 0; + } + if (ret < 0) + hid_err(hdev, "unable to request touch data (%d)\n", ret); + } + /* + * MT enable is usually not required after the first time, so don't + * consider it fatal. + */ + + /* + * For Apple Silicon trackpads, we want to query the dimensions on + * device open. This is because doing so requires the firmware, but + * we don't want to force a firmware load until the device is opened + * for the first time. So do that here and update the input properties + * just in time before userspace queries them. + */ + if (msc->query_dimensions) { + struct input_dev *input = msc->input; + u8 buf[32]; + struct { + __le32 width; + __le32 height; + __le16 min_x; + __le16 min_y; + __le16 max_x; + __le16 max_y; + } dim; + uint32_t x_span, y_span; + + ret = hid_hw_raw_request(hdev, SENSOR_DIMENSIONS_REPORT_ID, buf, sizeof(buf), HID_FEATURE_REPORT, HID_REQ_GET_REPORT); + if (ret < (int)(1 + sizeof(dim))) { + hid_err(hdev, "unable to request dimensions (%d)\n", ret); + return ret; + } + + memcpy(&dim, buf + 1, sizeof(dim)); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, + le16_to_int(dim.min_x), le16_to_int(dim.max_x), 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, + -le16_to_int(dim.max_y), -le16_to_int(dim.min_y), 0, 0); + x_span = le16_to_int(dim.max_x) - le16_to_int(dim.min_x); + y_span = le16_to_int(dim.max_y) - le16_to_int(dim.min_y); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, 100 * x_span / le32_to_cpu(dim.width) ); + input_abs_set_res(input, ABS_MT_POSITION_Y, 100 * y_span / le32_to_cpu(dim.height) ); + + /* copy info, as input_mt_init_slots() does */ + dev->absinfo[ABS_X] = dev->absinfo[ABS_MT_POSITION_X]; + dev->absinfo[ABS_Y] = dev->absinfo[ABS_MT_POSITION_Y]; + + msc->query_dimensions = false; + } + + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -192,7 +496,7 @@ static void magicmouse_emit_buttons(struct magicmouse_sc *msc, int state) } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -255,8 +559,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small @@ -385,6 +689,14 @@ static int magicmouse_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; @@ -515,6 +827,191 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + * @crc16: on last finger: crc over the whole message struct + * (i.e. message header + this struct) minus the last + * @crc16 field; unknown on all other fingers. + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * vendor trackpad report + * + * @num_fingers: the number of fingers being reported in @fingers + * @buttons: same as HID buttons + */ +struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; +}; + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + INTERNAL_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_mtp(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != TRACKPAD2_USB_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + +static int magicmouse_raw_event_t2(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] != TRACKPAD2_USB_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -532,7 +1029,17 @@ static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -610,7 +1117,7 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, @@ -689,6 +1196,171 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + return 0; +} + +struct magicmouse_t2_properties { + u32 id; + int min_x; + int min_y; + int max_x; + int max_y; + int res_x; + int res_y; +}; + +static const struct magicmouse_t2_properties magicmouse_t2_configs[] = { + T2_TOUCHPAD_ENTRY(J140K), + T2_TOUCHPAD_ENTRY(J132), + T2_TOUCHPAD_ENTRY(J680), + T2_TOUCHPAD_ENTRY(J680_ALT), + T2_TOUCHPAD_ENTRY(J213), + T2_TOUCHPAD_ENTRY(J214K), + T2_TOUCHPAD_ENTRY(J223), + T2_TOUCHPAD_ENTRY(J230K), + T2_TOUCHPAD_ENTRY(J152F), +}; + +static int magicmouse_setup_input_int_tpd(struct input_dev *input, + struct hid_device *hdev, int min_x, int min_y, + int max_x, int max_y, int res_x, int res_y, + bool query_dimensions) +{ + int error; + int mt_flags = 0; + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -INTERNAL_TP_MAX_FINGER_ORIENTATION, + INTERNAL_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, min_x, max_x, 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -max_y, -min_y, 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, res_x); + input_abs_set_res(input, ABS_MT_POSITION_Y, res_y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + * + * This also takes care of the dimensions query. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + msc->query_dimensions = query_dimensions; + + return 0; +} + +static int magicmouse_setup_input_mtp(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_int_tpd(input, hdev, J314_TP_MIN_X, + J314_TP_MIN_Y, J314_TP_MAX_X, + J314_TP_MAX_Y, J314_TP_RES_X, + J314_TP_RES_Y, true); + if (ret) + return ret; + + return 0; +} + +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_int_tpd(input, hdev, J314_TP_MIN_X, + J314_TP_MIN_Y, J314_TP_MAX_X, + J314_TP_MAX_Y, J314_TP_RES_X, + J314_TP_RES_Y, true); + if (ret) + return ret; + + return 0; +} + +static int magicmouse_setup_input_t2(struct input_dev *input, + struct hid_device *hdev) +{ + int min_x, min_y, max_x, max_y, res_x, res_y; + + for (size_t i = 0; i < ARRAY_SIZE(magicmouse_t2_configs); i++) { + if (magicmouse_t2_configs[i].id == hdev->product) { + min_x = magicmouse_t2_configs[i].min_x; + min_y = magicmouse_t2_configs[i].min_y; + max_x = magicmouse_t2_configs[i].max_x; + max_y = magicmouse_t2_configs[i].max_y; + res_x = magicmouse_t2_configs[i].res_x; + res_y = magicmouse_t2_configs[i].res_y; + } + } + + int ret = magicmouse_setup_input_int_tpd(input, hdev, min_x, min_y, + max_x, max_y, res_x, res_y, false); + if (ret) + return ret; + return 0; } @@ -730,55 +1402,6 @@ static int magicmouse_input_configured(struct hid_device *hdev, return 0; } -static int magicmouse_enable_multitouch(struct hid_device *hdev) -{ - const u8 *feature; - const u8 feature_mt[] = { 0xD7, 0x01 }; - const u8 feature_mt_mouse2[] = { 0xF1, 0x02, 0x01 }; - const u8 feature_mt_trackpad2_usb[] = { 0x02, 0x01 }; - const u8 feature_mt_trackpad2_bt[] = { 0xF1, 0x02, 0x01 }; - u8 *buf; - int ret; - int feature_size; - - if (hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || - hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC) { - if (hdev->vendor == BT_VENDOR_ID_APPLE) { - feature_size = sizeof(feature_mt_trackpad2_bt); - feature = feature_mt_trackpad2_bt; - } else { /* USB_VENDOR_ID_APPLE */ - feature_size = sizeof(feature_mt_trackpad2_usb); - feature = feature_mt_trackpad2_usb; - } - } else if (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { - feature_size = sizeof(feature_mt_mouse2); - feature = feature_mt_mouse2; - } else { - feature_size = sizeof(feature_mt); - feature = feature_mt; - } - - buf = kmemdup(feature, feature_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - ret = hid_hw_raw_request(hdev, buf[0], buf, feature_size, - HID_FEATURE_REPORT, HID_REQ_SET_REPORT); - kfree(buf); - return ret; -} - -static void magicmouse_enable_mt_work(struct work_struct *work) -{ - struct magicmouse_sc *msc = - container_of(work, struct magicmouse_sc, work.work); - int ret; - - ret = magicmouse_enable_multitouch(msc->hdev); - if (ret < 0) - hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); -} - static int magicmouse_fetch_battery(struct hid_device *hdev) { #ifdef CONFIG_HID_BATTERY_STRENGTH @@ -825,12 +1448,62 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + + switch (id->product) { + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + if (hdev->type != HID_TYPE_USBMOUSE) + return -ENODEV; + break; + } + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + switch (id->bus) { + case BUS_HOST: + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; + break; + case BUS_SPI: + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + break; + default: + switch (id->product) { + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: + msc->input_ops.raw_event = magicmouse_raw_event_t2; + msc->input_ops.setup_input = magicmouse_setup_input_t2; + break; + default: + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } + } + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); @@ -868,25 +1541,51 @@ static int magicmouse_probe(struct hid_device *hdev, goto err_stop_hw; } - if (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE) - report = hid_register_report(hdev, HID_INPUT_REPORT, - MOUSE_REPORT_ID, 0); - else if (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) - report = hid_register_report(hdev, HID_INPUT_REPORT, - MOUSE2_REPORT_ID, 0); - else if (id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || - id->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC) { - if (id->vendor == BT_VENDOR_ID_APPLE) - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD2_BT_REPORT_ID, 0); - else /* USB_VENDOR_ID_APPLE */ + switch (id->bus) { + case BUS_SPI: + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + break; + case BUS_HOST: + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + break; + default: + switch (id->product) { + case USB_DEVICE_ID_APPLE_MAGICMOUSE: + report = hid_register_report(hdev, HID_INPUT_REPORT, MOUSE_REPORT_ID, 0); + break; + case USB_DEVICE_ID_APPLE_MAGICMOUSE2: + report = hid_register_report(hdev, HID_INPUT_REPORT, MOUSE2_REPORT_ID, 0); + break; + case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: + case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: + switch (id->vendor) { + case BT_VENDOR_ID_APPLE: + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD2_BT_REPORT_ID, 0); + break; + default: + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD2_USB_REPORT_ID, 0); + } + break; + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K: + case USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F: report = hid_register_report(hdev, HID_INPUT_REPORT, TRACKPAD2_USB_REPORT_ID, 0); - } else { /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD_REPORT_ID, 0); - report = hid_register_report(hdev, HID_INPUT_REPORT, - DOUBLE_REPORT_ID, 0); + break; + default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD_REPORT_ID, 0); + report = hid_register_report(hdev, HID_INPUT_REPORT, + DOUBLE_REPORT_ID, 0); + } } if (!report) { @@ -896,21 +1595,14 @@ static int magicmouse_probe(struct hid_device *hdev, } report->size = 6; - /* - * Some devices repond with 'invalid report id' when feature - * report switching it into multitouch mode is sent to it. - * - * This results in -EIO from the _raw low-level transport callback, - * but there seems to be no other way of switching the mode. - * Thus the super-ugly hacky success check below. - */ - ret = magicmouse_enable_multitouch(hdev); - if (ret != -EIO && ret < 0) { - hid_err(hdev, "unable to request touch data (%d)\n", ret); - goto err_stop_hw; - } - if (ret == -EIO && id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { - schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + + /* SPI devices need to watch for reset events to re-send the MT enable */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_RESET_REPORT_ID, 0); + report->size = 2; } return 0; @@ -981,10 +1673,42 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K), .driver_data = 0 }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + if (hdev->bus == BUS_SPI) + return magicmouse_enable_multitouch(hdev); + + return 0; +} +#endif + static struct hid_driver magicmouse_driver = { .name = "magicmouse", .id_table = magic_mice, @@ -995,6 +1719,10 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif + }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 7ac8e16e6158..7a64e45d5a0d 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -73,6 +73,7 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_FORCE_MULTI_INPUT BIT(20) #define MT_QUIRK_DISABLE_WAKEUP BIT(21) #define MT_QUIRK_ORIENTATION_INVERT BIT(22) +#define MT_QUIRK_TOUCH_IS_TIPSTATE BIT(23) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -153,6 +154,7 @@ struct mt_class { __s32 sn_height; /* Signal/noise ratio for height events */ __s32 sn_pressure; /* Signal/noise ratio for pressure events */ __u8 maxcontacts; + bool is_direct; /* true for touchscreens */ bool is_indirect; /* true for touchpads */ bool export_all_inputs; /* do not ignore mouse, keyboards, etc... */ }; @@ -220,6 +222,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_GOOGLE 0x0111 #define MT_CLS_RAZER_BLADE_STEALTH 0x0112 #define MT_CLS_SMART_TECH 0x0113 +#define MT_CLS_APPLE_TOUCHBAR 0x0114 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -405,6 +408,13 @@ static const struct mt_class mt_classes[] = { MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_SEPARATE_APP_REPORT, }, + { .name = MT_CLS_APPLE_TOUCHBAR, + .quirks = MT_QUIRK_HOVERING | + MT_QUIRK_TOUCH_IS_TIPSTATE | + MT_QUIRK_SLOT_IS_CONTACTID_MINUS_ONE, + .is_direct = true, + .maxcontacts = 11, + }, { .name = MT_CLS_SIS, .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | MT_QUIRK_ALWAYS_VALID | @@ -503,9 +513,6 @@ static void mt_feature_mapping(struct hid_device *hdev, if (!td->maxcontacts && field->logical_maximum <= MT_MAX_MAXCONTACT) td->maxcontacts = field->logical_maximum; - if (td->mtclass.maxcontacts) - /* check if the maxcontacts is given by the class */ - td->maxcontacts = td->mtclass.maxcontacts; break; case HID_DG_BUTTONTYPE: @@ -579,13 +586,13 @@ static struct mt_application *mt_allocate_application(struct mt_device *td, mt_application->application = application; INIT_LIST_HEAD(&mt_application->mt_usages); - if (application == HID_DG_TOUCHSCREEN) + if (application == HID_DG_TOUCHSCREEN && !td->mtclass.is_indirect) mt_application->mt_flags |= INPUT_MT_DIRECT; /* * Model touchscreens providing buttons as touchpads. */ - if (application == HID_DG_TOUCHPAD) { + if (application == HID_DG_TOUCHPAD && !td->mtclass.is_direct) { mt_application->mt_flags |= INPUT_MT_POINTER; td->inputmode_value = MT_INPUTMODE_TOUCHPAD; } @@ -649,7 +656,9 @@ static struct mt_report_data *mt_allocate_report_data(struct mt_device *td, if (field->logical == HID_DG_FINGER || td->hdev->group != HID_GROUP_MULTITOUCH_WIN_8) { for (n = 0; n < field->report_count; n++) { - if (field->usage[n].hid == HID_DG_CONTACTID) { + unsigned int hid = field->usage[n].hid; + + if (hid == HID_DG_CONTACTID || hid == HID_DG_TRANSDUCER_INDEX) { rdata->is_mt_collection = true; break; } @@ -821,6 +830,15 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, MT_STORE_FIELD(confidence_state); return 1; + case HID_DG_TOUCH: + /* + * Legacy devices use TIPSWITCH and not TOUCH. + * Let's just ignore this field unless the quirk is set. + */ + if (!(cls->quirks & MT_QUIRK_TOUCH_IS_TIPSTATE)) + return -1; + + fallthrough; case HID_DG_TIPSWITCH: if (field->application != HID_GD_SYSTEM_MULTIAXIS) input_set_capability(hi->input, @@ -828,6 +846,7 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, MT_STORE_FIELD(tip_state); return 1; case HID_DG_CONTACTID: + case HID_DG_TRANSDUCER_INDEX: MT_STORE_FIELD(contactid); app->touches_by_report++; return 1; @@ -883,10 +902,6 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi, case HID_DG_CONTACTMAX: /* contact max are global to the report */ return -1; - case HID_DG_TOUCH: - /* Legacy devices use TIPSWITCH and not TOUCH. - * Let's just ignore this field. */ - return -1; } /* let hid-input decide for the others */ return 0; @@ -1314,6 +1329,10 @@ static int mt_touch_input_configured(struct hid_device *hdev, struct input_dev *input = hi->input; int ret; + /* check if the maxcontacts is given by the class */ + if (cls->maxcontacts) + td->maxcontacts = cls->maxcontacts; + if (!td->maxcontacts) td->maxcontacts = MT_DEFAULT_MAXCONTACT; @@ -1321,6 +1340,9 @@ static int mt_touch_input_configured(struct hid_device *hdev, if (td->serial_maybe) mt_post_parse_default_settings(td, app); + if (cls->is_direct) + app->mt_flags |= INPUT_MT_DIRECT; + if (cls->is_indirect) app->mt_flags |= INPUT_MT_POINTER; @@ -1772,6 +1794,15 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id) } } + ret = hid_parse(hdev); + if (ret != 0) + return ret; + + if (mtclass->name == MT_CLS_APPLE_TOUCHBAR && + !hid_find_field(hdev, HID_INPUT_REPORT, + HID_DG_TOUCHPAD, HID_DG_TRANSDUCER_INDEX)) + return -ENODEV; + td = devm_kzalloc(&hdev->dev, sizeof(struct mt_device), GFP_KERNEL); if (!td) { dev_err(&hdev->dev, "cannot allocate multitouch data\n"); @@ -1819,10 +1850,6 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id) timer_setup(&td->release_timer, mt_expired_timeout, 0); - ret = hid_parse(hdev); - if (ret != 0) - return ret; - if (mtclass->quirks & MT_QUIRK_FIX_CONST_CONTACT_ID) mt_fix_const_fields(hdev, HID_DG_CONTACTID); @@ -2304,6 +2331,11 @@ static const struct hid_device_id mt_devices[] = { MT_USB_DEVICE(USB_VENDOR_ID_XIROKU, USB_DEVICE_ID_XIROKU_CSR2) }, + /* Apple Touch Bars */ + { .driver_data = MT_CLS_APPLE_TOUCHBAR, + HID_USB_DEVICE(USB_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, + /* Google MT devices */ { .driver_data = MT_CLS_GOOGLE, HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_GOOGLE, diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 7fefeb413ec3..1fba3b7f6d41 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -314,6 +314,7 @@ static const struct hid_device_id hid_have_special_driver[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680_ALT) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223) }, @@ -969,14 +970,6 @@ static const struct hid_device_id hid_mouse_ignore_list[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ISO) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J140K) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J132) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J680) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J213) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J214K) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J223) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J230K) }, - { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRINGT2_J152F) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) }, { } diff --git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig new file mode 100644 index 000000000000..8e37f0fec28a --- /dev/null +++ b/drivers/hid/spi-hid/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + default ARCH_APPLE + depends on SPI && INPUT && OF + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + default y if SPI_HID_APPLE_OF=y + default m if SPI_HID_APPLE_OF=m + select HID + select CRC16 diff --git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile new file mode 100644 index 000000000000..f276ee12cb94 --- /dev/null +++ b/drivers/hid/spi-hid/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c new file mode 100644 index 000000000000..1f8fa64d6d86 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -0,0 +1,1194 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + u8 reply_buf[SPIHID_DESC_MAX]; + u32 reply_len; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @msgid: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + int ret; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + if (rtype != HID_FEATURE_REPORT) + return -EINVAL; + + idev->reply_len = 0; + ret = spihid_apple_request(spihid, idev->id, 0x32, reportnum, 0x00, len, NULL, 0); + if (ret < 0) + return ret; + + ret = wait_event_interruptible_timeout(spihid->wait, idev->reply_len, + SPIHID_DEF_WAIT); + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) { + dev_err(&spihid->spidev->dev, "waiting for get report failed: %d", ret); + return ret; + } + memcpy(buf, idev->reply_buf, max_t(size_t, len, idev->reply_len)); + return idev->reply_len; + + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, + .max_buffer_size = SPIHID_MAX_INPUT_REPORT_SIZE, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_iface_get_report(struct spihid_apple *spihid, + u32 device, u8 report, + u8 *payload, size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, device); + + if (!iface) + return false; + + if (len > sizeof(iface->reply_buf) || len < 1) + return false; + + memcpy(iface->reply_buf, payload, len); + iface->reply_len = len; + + wake_up_interruptible(&spihid->wait); + + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + if (hdr->unknown0 == 0x32) { + return spihid_process_iface_get_report(spihid, device, hdr->unknown1, payload, len); + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + size_t payload_len; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + payload_len = le16_to_cpu(hdr->length); + + if (payload_len == 0 || + (payload_len + sizeof(struct spihid_msg_hdr) + 2) > length) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, payload_len); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, device, hdr, payload, + payload_len); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + char *suffix; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + /* + * Use 'Apple SPI Keyboard' and 'Apple SPI Trackpad' as input device + * names. The device names need to be distinct since at least Kwin uses + * the tripple Vendor ID, Product ID, Name to identify devices. + */ + snprintf(hid->name, sizeof(hid->name), "Apple SPI %s", iface->name); + // strip ' / Boot' suffix from the name + suffix = strstr(hid->name, " / Boot"); + if (suffix) + suffix[0] = '\0'; + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* + * allocate SPI buffers + * Overallocate the receice buffer since it passed directly into + * hid_input_report / hid_report_raw_event. The later expects the buffer + * to be HID_MAX_BUFFER_SIZE (16k) or hid_ll_driver.max_buffer_size if + * set. + */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c new file mode 100644 index 000000000000..3f87b299351d --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -0,0 +1,151 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + .pm = &spihid_apple_core_pm, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h new file mode 100644 index 000000000000..9abecd1ba780 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +extern const struct dev_pm_ops spihid_apple_core_pm; + +#endif /* SPI_HID_APPLE_H */ diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c index fc6d6a9053ce..698f44794453 100644 --- a/drivers/hwmon/applesmc.c +++ b/drivers/hwmon/applesmc.c @@ -6,6 +6,7 @@ * * Copyright (C) 2007 Nicolas Boichat * Copyright (C) 2010 Henrik Rydberg + * Copyright (C) 2019 Paul Pawlowski * * Based on hdaps.c driver: * Copyright (C) 2005 Robert Love @@ -18,7 +19,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include -#include +#include #include #include #include @@ -35,12 +36,24 @@ #include /* data port used by Apple SMC */ -#define APPLESMC_DATA_PORT 0x300 +#define APPLESMC_DATA_PORT 0 /* command/status port used by Apple SMC */ -#define APPLESMC_CMD_PORT 0x304 +#define APPLESMC_CMD_PORT 4 #define APPLESMC_NR_PORTS 32 /* 0x300-0x31f */ +#define APPLESMC_IOMEM_KEY_DATA 0 +#define APPLESMC_IOMEM_KEY_STATUS 0x4005 +#define APPLESMC_IOMEM_KEY_NAME 0x78 +#define APPLESMC_IOMEM_KEY_DATA_LEN 0x7D +#define APPLESMC_IOMEM_KEY_SMC_ID 0x7E +#define APPLESMC_IOMEM_KEY_CMD 0x7F +#define APPLESMC_IOMEM_MIN_SIZE 0x4006 + +#define APPLESMC_IOMEM_KEY_TYPE_CODE 0 +#define APPLESMC_IOMEM_KEY_TYPE_DATA_LEN 5 +#define APPLESMC_IOMEM_KEY_TYPE_FLAGS 6 + #define APPLESMC_MAX_DATA_LENGTH 32 /* Apple SMC status bits */ @@ -74,6 +87,7 @@ #define FAN_ID_FMT "F%dID" /* r-o char[16] */ #define TEMP_SENSOR_TYPE "sp78" +#define FLOAT_TYPE "flt " /* List of keys used to read/write fan speeds */ static const char *const fan_speed_fmt[] = { @@ -83,6 +97,7 @@ static const char *const fan_speed_fmt[] = { "F%dSf", /* safe speed - not all models */ "F%dTg", /* target speed (manual: rw) */ }; +#define FAN_MANUAL_FMT "F%dMd" #define INIT_TIMEOUT_MSECS 5000 /* wait up to 5s for device init ... */ #define INIT_WAIT_MSECS 50 /* ... in 50ms increments */ @@ -119,7 +134,7 @@ struct applesmc_entry { }; /* Register lookup and registers common to all SMCs */ -static struct applesmc_registers { +struct applesmc_registers { struct mutex mutex; /* register read/write mutex */ unsigned int key_count; /* number of SMC registers */ unsigned int fan_count; /* number of fans */ @@ -133,26 +148,38 @@ static struct applesmc_registers { bool init_complete; /* true when fully initialized */ struct applesmc_entry *cache; /* cached key entries */ const char **index; /* temperature key index */ -} smcreg = { - .mutex = __MUTEX_INITIALIZER(smcreg.mutex), }; -static const int debug; -static struct platform_device *pdev; -static s16 rest_x; -static s16 rest_y; -static u8 backlight_state[2]; +struct applesmc_device { + struct acpi_device *dev; + struct device *ldev; + struct applesmc_registers reg; -static struct device *hwmon_dev; -static struct input_dev *applesmc_idev; + bool port_base_set, iomem_base_set; + u16 port_base; + u8 *__iomem iomem_base; + u32 iomem_base_addr, iomem_base_size; -/* - * Last index written to key_at_index sysfs file, and value to use for all other - * key_at_index_* sysfs files. - */ -static unsigned int key_at_index; + s16 rest_x; + s16 rest_y; + + u8 backlight_state[2]; + + struct device *hwmon_dev; + struct input_dev *idev; + + /* + * Last index written to key_at_index sysfs file, and value to use for all other + * key_at_index_* sysfs files. + */ + unsigned int key_at_index; -static struct workqueue_struct *applesmc_led_wq; + struct workqueue_struct *backlight_wq; + struct work_struct backlight_work; + struct led_classdev backlight_dev; +}; + +static const int debug; /* * Wait for specific status bits with a mask on the SMC. @@ -162,7 +189,7 @@ static struct workqueue_struct *applesmc_led_wq; * run out past 500ms. */ -static int wait_status(u8 val, u8 mask) +static int port_wait_status(struct applesmc_device *smc, u8 val, u8 mask) { u8 status; int us; @@ -170,7 +197,7 @@ static int wait_status(u8 val, u8 mask) us = APPLESMC_MIN_WAIT; for (i = 0; i < 24 ; i++) { - status = inb(APPLESMC_CMD_PORT); + status = inb(smc->port_base + APPLESMC_CMD_PORT); if ((status & mask) == val) return 0; usleep_range(us, us * 2); @@ -180,13 +207,13 @@ static int wait_status(u8 val, u8 mask) return -EIO; } -/* send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ +/* port_send_byte - Write to SMC data port. Callers must hold applesmc_lock. */ -static int send_byte(u8 cmd, u16 port) +static int port_send_byte(struct applesmc_device *smc, u8 cmd, u16 port) { int status; - status = wait_status(0, SMC_STATUS_IB_CLOSED); + status = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); if (status) return status; /* @@ -195,24 +222,25 @@ static int send_byte(u8 cmd, u16 port) * this extra read may not happen if status returns both * simultaneously and this would appear to be required. */ - status = wait_status(SMC_STATUS_BUSY, SMC_STATUS_BUSY); + status = port_wait_status(smc, SMC_STATUS_BUSY, SMC_STATUS_BUSY); if (status) return status; - outb(cmd, port); + outb(cmd, smc->port_base + port); return 0; } -/* send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ +/* port_send_command - Write a command to the SMC. Callers must hold applesmc_lock. */ -static int send_command(u8 cmd) +static int port_send_command(struct applesmc_device *smc, u8 cmd) { int ret; - ret = wait_status(0, SMC_STATUS_IB_CLOSED); + ret = port_wait_status(smc, 0, SMC_STATUS_IB_CLOSED); if (ret) return ret; - outb(cmd, APPLESMC_CMD_PORT); + + outb(cmd, smc->port_base + APPLESMC_CMD_PORT); return 0; } @@ -222,110 +250,304 @@ static int send_command(u8 cmd) * If busy is stuck high after the command then the SMC is jammed. */ -static int smc_sane(void) +static int port_smc_sane(struct applesmc_device *smc) { int ret; - ret = wait_status(0, SMC_STATUS_BUSY); + ret = port_wait_status(smc, 0, SMC_STATUS_BUSY); if (!ret) return ret; - ret = send_command(APPLESMC_READ_CMD); + ret = port_send_command(smc, APPLESMC_READ_CMD); if (ret) return ret; - return wait_status(0, SMC_STATUS_BUSY); + return port_wait_status(smc, 0, SMC_STATUS_BUSY); } -static int send_argument(const char *key) +static int port_send_argument(struct applesmc_device *smc, const char *key) { int i; for (i = 0; i < 4; i++) - if (send_byte(key[i], APPLESMC_DATA_PORT)) + if (port_send_byte(smc, key[i], APPLESMC_DATA_PORT)) return -EIO; return 0; } -static int read_smc(u8 cmd, const char *key, u8 *buffer, u8 len) +static int port_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, + u8 *buffer, u8 len) { u8 status, data = 0; int i; int ret; - ret = smc_sane(); + ret = port_smc_sane(smc); if (ret) return ret; - if (send_command(cmd) || send_argument(key)) { + if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { pr_warn("%.4s: read arg fail\n", key); return -EIO; } /* This has no effect on newer (2012) SMCs */ - if (send_byte(len, APPLESMC_DATA_PORT)) { + if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { pr_warn("%.4s: read len fail\n", key); return -EIO; } for (i = 0; i < len; i++) { - if (wait_status(SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, + if (port_wait_status(smc, + SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY, SMC_STATUS_AWAITING_DATA | SMC_STATUS_BUSY)) { pr_warn("%.4s: read data[%d] fail\n", key, i); return -EIO; } - buffer[i] = inb(APPLESMC_DATA_PORT); + buffer[i] = inb(smc->port_base + APPLESMC_DATA_PORT); } /* Read the data port until bit0 is cleared */ for (i = 0; i < 16; i++) { udelay(APPLESMC_MIN_WAIT); - status = inb(APPLESMC_CMD_PORT); + status = inb(smc->port_base + APPLESMC_CMD_PORT); if (!(status & SMC_STATUS_AWAITING_DATA)) break; - data = inb(APPLESMC_DATA_PORT); + data = inb(smc->port_base + APPLESMC_DATA_PORT); } if (i) pr_warn("flushed %d bytes, last value is: %d\n", i, data); - return wait_status(0, SMC_STATUS_BUSY); + return port_wait_status(smc, 0, SMC_STATUS_BUSY); } -static int write_smc(u8 cmd, const char *key, const u8 *buffer, u8 len) +static int port_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, + const u8 *buffer, u8 len) { int i; int ret; - ret = smc_sane(); + ret = port_smc_sane(smc); if (ret) return ret; - if (send_command(cmd) || send_argument(key)) { + if (port_send_command(smc, cmd) || port_send_argument(smc, key)) { pr_warn("%s: write arg fail\n", key); return -EIO; } - if (send_byte(len, APPLESMC_DATA_PORT)) { + if (port_send_byte(smc, len, APPLESMC_DATA_PORT)) { pr_warn("%.4s: write len fail\n", key); return -EIO; } for (i = 0; i < len; i++) { - if (send_byte(buffer[i], APPLESMC_DATA_PORT)) { + if (port_send_byte(smc, buffer[i], APPLESMC_DATA_PORT)) { pr_warn("%s: write data fail\n", key); return -EIO; } } - return wait_status(0, SMC_STATUS_BUSY); + return port_wait_status(smc, 0, SMC_STATUS_BUSY); } -static int read_register_count(unsigned int *count) +static int port_get_smc_key_info(struct applesmc_device *smc, + const char *key, struct applesmc_entry *info) { - __be32 be; int ret; + u8 raw[6]; - ret = read_smc(APPLESMC_READ_CMD, KEY_COUNT_KEY, (u8 *)&be, 4); + ret = port_read_smc(smc, APPLESMC_GET_KEY_TYPE_CMD, key, raw, 6); if (ret) return ret; + info->len = raw[0]; + memcpy(info->type, &raw[1], 4); + info->flags = raw[5]; + return 0; +} + + +/* + * MMIO based communication. + * TODO: Use updated mechanism for cmd timeout/retry + */ + +static void iomem_clear_status(struct applesmc_device *smc) +{ + if (ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS)) + iowrite8(0, smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); +} + +static int iomem_wait_read(struct applesmc_device *smc) +{ + u8 status; + int us; + int i; + + us = APPLESMC_MIN_WAIT; + for (i = 0; i < 24 ; i++) { + status = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); + if (status & 0x20) + return 0; + usleep_range(us, us * 2); + if (i > 9) + us <<= 1; + } + + dev_warn(smc->ldev, "%s... timeout\n", __func__); + return -EIO; +} + +static int iomem_read_smc(struct applesmc_device *smc, u8 cmd, const char *key, + u8 *buffer, u8 len) +{ + u8 err, remote_len; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "read_smc_mmio(%x %8x/%.4s) failed: %u\n", + cmd, key_int, key, err); + return -EIO; + } + + if (cmd == APPLESMC_READ_CMD) { + remote_len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); + if (remote_len != len) { + dev_warn(smc->ldev, + "read_smc_mmio(%x %8x/%.4s) failed: buffer length mismatch (remote = %u, requested = %u)\n", + cmd, key_int, key, remote_len, len); + return -EINVAL; + } + } else { + remote_len = len; + } + + memcpy_fromio(buffer, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, + remote_len); + + dev_dbg(smc->ldev, "read_smc_mmio(%x %8x/%.4s): buflen=%u reslen=%u\n", + cmd, key_int, key, len, remote_len); + print_hex_dump_bytes("read_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, remote_len); + return 0; +} + +static int iomem_get_smc_key_type(struct applesmc_device *smc, const char *key, + struct applesmc_entry *e) +{ + u8 err; + u8 cmd = APPLESMC_GET_KEY_TYPE_CMD; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "get_smc_key_type_mmio(%.4s) failed: %u\n", key, err); + return -EIO; + } + + e->len = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_DATA_LEN); + *((uint32_t *) e->type) = ioread32( + smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_CODE); + e->flags = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_TYPE_FLAGS); + + dev_dbg(smc->ldev, "get_smc_key_type_mmio(%.4s): len=%u type=%.4s flags=%x\n", + key, e->len, e->type, e->flags); + return 0; +} + +static int iomem_write_smc(struct applesmc_device *smc, u8 cmd, const char *key, + const u8 *buffer, u8 len) +{ + u8 err; + u32 key_int = *((u32 *) key); + + iomem_clear_status(smc); + iowrite32(key_int, smc->iomem_base + APPLESMC_IOMEM_KEY_NAME); + memcpy_toio(smc->iomem_base + APPLESMC_IOMEM_KEY_DATA, buffer, len); + iowrite32(len, smc->iomem_base + APPLESMC_IOMEM_KEY_DATA_LEN); + iowrite32(0, smc->iomem_base + APPLESMC_IOMEM_KEY_SMC_ID); + iowrite32(cmd, smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + + if (iomem_wait_read(smc)) + return -EIO; + + err = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_CMD); + if (err != 0) { + dev_warn(smc->ldev, "write_smc_mmio(%x %.4s) failed: %u\n", cmd, key, err); + print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); + return -EIO; + } + + dev_dbg(smc->ldev, "write_smc_mmio(%x %.4s): buflen=%u\n", cmd, key, len); + print_hex_dump_bytes("write_smc_mmio(): ", DUMP_PREFIX_NONE, buffer, len); + return 0; +} + + +static int read_smc(struct applesmc_device *smc, const char *key, + u8 *buffer, u8 len) +{ + if (smc->iomem_base_set) + return iomem_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); + else + return port_read_smc(smc, APPLESMC_READ_CMD, key, buffer, len); +} + +static int write_smc(struct applesmc_device *smc, const char *key, + const u8 *buffer, u8 len) +{ + if (smc->iomem_base_set) + return iomem_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); + else + return port_write_smc(smc, APPLESMC_WRITE_CMD, key, buffer, len); +} + +static int get_smc_key_by_index(struct applesmc_device *smc, + unsigned int index, char *key) +{ + __be32 be; + + be = cpu_to_be32(index); + if (smc->iomem_base_set) + return iomem_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, + (const char *) &be, (u8 *) key, 4); + else + return port_read_smc(smc, APPLESMC_GET_KEY_BY_INDEX_CMD, + (const char *) &be, (u8 *) key, 4); +} + +static int get_smc_key_info(struct applesmc_device *smc, const char *key, + struct applesmc_entry *info) +{ + if (smc->iomem_base_set) + return iomem_get_smc_key_type(smc, key, info); + else + return port_get_smc_key_info(smc, key, info); +} + +static int read_register_count(struct applesmc_device *smc, + unsigned int *count) +{ + __be32 be; + int ret; + + ret = read_smc(smc, KEY_COUNT_KEY, (u8 *)&be, 4); + if (ret < 0) + return ret; *count = be32_to_cpu(be); return 0; @@ -338,76 +560,73 @@ static int read_register_count(unsigned int *count) * All functions below are concurrency safe - callers should NOT hold lock. */ -static int applesmc_read_entry(const struct applesmc_entry *entry, - u8 *buf, u8 len) +static int applesmc_read_entry(struct applesmc_device *smc, + const struct applesmc_entry *entry, u8 *buf, u8 len) { int ret; if (entry->len != len) return -EINVAL; - mutex_lock(&smcreg.mutex); - ret = read_smc(APPLESMC_READ_CMD, entry->key, buf, len); - mutex_unlock(&smcreg.mutex); + mutex_lock(&smc->reg.mutex); + ret = read_smc(smc, entry->key, buf, len); + mutex_unlock(&smc->reg.mutex); return ret; } -static int applesmc_write_entry(const struct applesmc_entry *entry, - const u8 *buf, u8 len) +static int applesmc_write_entry(struct applesmc_device *smc, + const struct applesmc_entry *entry, const u8 *buf, u8 len) { int ret; if (entry->len != len) return -EINVAL; - mutex_lock(&smcreg.mutex); - ret = write_smc(APPLESMC_WRITE_CMD, entry->key, buf, len); - mutex_unlock(&smcreg.mutex); + mutex_lock(&smc->reg.mutex); + ret = write_smc(smc, entry->key, buf, len); + mutex_unlock(&smc->reg.mutex); return ret; } -static const struct applesmc_entry *applesmc_get_entry_by_index(int index) +static const struct applesmc_entry *applesmc_get_entry_by_index( + struct applesmc_device *smc, int index) { - struct applesmc_entry *cache = &smcreg.cache[index]; - u8 key[4], info[6]; - __be32 be; + struct applesmc_entry *cache = &smc->reg.cache[index]; + char key[4]; int ret = 0; if (cache->valid) return cache; - mutex_lock(&smcreg.mutex); + mutex_lock(&smc->reg.mutex); if (cache->valid) goto out; - be = cpu_to_be32(index); - ret = read_smc(APPLESMC_GET_KEY_BY_INDEX_CMD, (u8 *)&be, key, 4); + ret = get_smc_key_by_index(smc, index, key); if (ret) goto out; - ret = read_smc(APPLESMC_GET_KEY_TYPE_CMD, key, info, 6); + memcpy(cache->key, key, 4); + + ret = get_smc_key_info(smc, key, cache); if (ret) goto out; - - memcpy(cache->key, key, 4); - cache->len = info[0]; - memcpy(cache->type, &info[1], 4); - cache->flags = info[5]; cache->valid = true; out: - mutex_unlock(&smcreg.mutex); + mutex_unlock(&smc->reg.mutex); if (ret) return ERR_PTR(ret); return cache; } -static int applesmc_get_lower_bound(unsigned int *lo, const char *key) +static int applesmc_get_lower_bound(struct applesmc_device *smc, + unsigned int *lo, const char *key) { - int begin = 0, end = smcreg.key_count; + int begin = 0, end = smc->reg.key_count; const struct applesmc_entry *entry; while (begin != end) { int middle = begin + (end - begin) / 2; - entry = applesmc_get_entry_by_index(middle); + entry = applesmc_get_entry_by_index(smc, middle); if (IS_ERR(entry)) { *lo = 0; return PTR_ERR(entry); @@ -422,16 +641,17 @@ static int applesmc_get_lower_bound(unsigned int *lo, const char *key) return 0; } -static int applesmc_get_upper_bound(unsigned int *hi, const char *key) +static int applesmc_get_upper_bound(struct applesmc_device *smc, + unsigned int *hi, const char *key) { - int begin = 0, end = smcreg.key_count; + int begin = 0, end = smc->reg.key_count; const struct applesmc_entry *entry; while (begin != end) { int middle = begin + (end - begin) / 2; - entry = applesmc_get_entry_by_index(middle); + entry = applesmc_get_entry_by_index(smc, middle); if (IS_ERR(entry)) { - *hi = smcreg.key_count; + *hi = smc->reg.key_count; return PTR_ERR(entry); } if (strcmp(key, entry->key) < 0) @@ -444,50 +664,54 @@ static int applesmc_get_upper_bound(unsigned int *hi, const char *key) return 0; } -static const struct applesmc_entry *applesmc_get_entry_by_key(const char *key) +static const struct applesmc_entry *applesmc_get_entry_by_key( + struct applesmc_device *smc, const char *key) { int begin, end; int ret; - ret = applesmc_get_lower_bound(&begin, key); + ret = applesmc_get_lower_bound(smc, &begin, key); if (ret) return ERR_PTR(ret); - ret = applesmc_get_upper_bound(&end, key); + ret = applesmc_get_upper_bound(smc, &end, key); if (ret) return ERR_PTR(ret); if (end - begin != 1) return ERR_PTR(-EINVAL); - return applesmc_get_entry_by_index(begin); + return applesmc_get_entry_by_index(smc, begin); } -static int applesmc_read_key(const char *key, u8 *buffer, u8 len) +static int applesmc_read_key(struct applesmc_device *smc, + const char *key, u8 *buffer, u8 len) { const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_key(key); + entry = applesmc_get_entry_by_key(smc, key); if (IS_ERR(entry)) return PTR_ERR(entry); - return applesmc_read_entry(entry, buffer, len); + return applesmc_read_entry(smc, entry, buffer, len); } -static int applesmc_write_key(const char *key, const u8 *buffer, u8 len) +static int applesmc_write_key(struct applesmc_device *smc, + const char *key, const u8 *buffer, u8 len) { const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_key(key); + entry = applesmc_get_entry_by_key(smc, key); if (IS_ERR(entry)) return PTR_ERR(entry); - return applesmc_write_entry(entry, buffer, len); + return applesmc_write_entry(smc, entry, buffer, len); } -static int applesmc_has_key(const char *key, bool *value) +static int applesmc_has_key(struct applesmc_device *smc, + const char *key, bool *value) { const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_key(key); + entry = applesmc_get_entry_by_key(smc, key); if (IS_ERR(entry) && PTR_ERR(entry) != -EINVAL) return PTR_ERR(entry); @@ -498,12 +722,13 @@ static int applesmc_has_key(const char *key, bool *value) /* * applesmc_read_s16 - Read 16-bit signed big endian register */ -static int applesmc_read_s16(const char *key, s16 *value) +static int applesmc_read_s16(struct applesmc_device *smc, + const char *key, s16 *value) { u8 buffer[2]; int ret; - ret = applesmc_read_key(key, buffer, 2); + ret = applesmc_read_key(smc, key, buffer, 2); if (ret) return ret; @@ -511,31 +736,68 @@ static int applesmc_read_s16(const char *key, s16 *value) return 0; } +/** + * applesmc_float_to_u32 - Retrieve the integral part of a float. + * This is needed because Apple made fans use float values in the T2. + * The fractional point is not significantly useful though, and the integral + * part can be easily extracted. + */ +static inline u32 applesmc_float_to_u32(u32 d) +{ + u8 sign = (u8) ((d >> 31) & 1); + s32 exp = (s32) ((d >> 23) & 0xff) - 0x7f; + u32 fr = d & ((1u << 23) - 1); + + if (sign || exp < 0) + return 0; + + return (u32) ((1u << exp) + (fr >> (23 - exp))); +} + +/** + * applesmc_u32_to_float - Convert an u32 into a float. + * See applesmc_float_to_u32 for a rationale. + */ +static inline u32 applesmc_u32_to_float(u32 d) +{ + u32 dc = d, bc = 0, exp; + + if (!d) + return 0; + + while (dc >>= 1) + ++bc; + exp = 0x7f + bc; + + return (u32) ((exp << 23) | + ((d << (23 - (exp - 0x7f))) & ((1u << 23) - 1))); +} /* * applesmc_device_init - initialize the accelerometer. Can sleep. */ -static void applesmc_device_init(void) +static void applesmc_device_init(struct applesmc_device *smc) { int total; u8 buffer[2]; - if (!smcreg.has_accelerometer) + if (!smc->reg.has_accelerometer) return; for (total = INIT_TIMEOUT_MSECS; total > 0; total -= INIT_WAIT_MSECS) { - if (!applesmc_read_key(MOTION_SENSOR_KEY, buffer, 2) && + if (!applesmc_read_key(smc, MOTION_SENSOR_KEY, buffer, 2) && (buffer[0] != 0x00 || buffer[1] != 0x00)) return; buffer[0] = 0xe0; buffer[1] = 0x00; - applesmc_write_key(MOTION_SENSOR_KEY, buffer, 2); + applesmc_write_key(smc, MOTION_SENSOR_KEY, buffer, 2); msleep(INIT_WAIT_MSECS); } pr_warn("failed to init the device\n"); } -static int applesmc_init_index(struct applesmc_registers *s) +static int applesmc_init_index(struct applesmc_device *smc, + struct applesmc_registers *s) { const struct applesmc_entry *entry; unsigned int i; @@ -548,7 +810,7 @@ static int applesmc_init_index(struct applesmc_registers *s) return -ENOMEM; for (i = s->temp_begin; i < s->temp_end; i++) { - entry = applesmc_get_entry_by_index(i); + entry = applesmc_get_entry_by_index(smc, i); if (IS_ERR(entry)) continue; if (strcmp(entry->type, TEMP_SENSOR_TYPE)) @@ -562,9 +824,9 @@ static int applesmc_init_index(struct applesmc_registers *s) /* * applesmc_init_smcreg_try - Try to initialize register cache. Idempotent. */ -static int applesmc_init_smcreg_try(void) +static int applesmc_init_smcreg_try(struct applesmc_device *smc) { - struct applesmc_registers *s = &smcreg; + struct applesmc_registers *s = &smc->reg; bool left_light_sensor = false, right_light_sensor = false; unsigned int count; u8 tmp[1]; @@ -573,7 +835,7 @@ static int applesmc_init_smcreg_try(void) if (s->init_complete) return 0; - ret = read_register_count(&count); + ret = read_register_count(smc, &count); if (ret) return ret; @@ -590,35 +852,35 @@ static int applesmc_init_smcreg_try(void) if (!s->cache) return -ENOMEM; - ret = applesmc_read_key(FANS_COUNT, tmp, 1); + ret = applesmc_read_key(smc, FANS_COUNT, tmp, 1); if (ret) return ret; s->fan_count = tmp[0]; if (s->fan_count > 10) s->fan_count = 10; - ret = applesmc_get_lower_bound(&s->temp_begin, "T"); + ret = applesmc_get_lower_bound(smc, &s->temp_begin, "T"); if (ret) return ret; - ret = applesmc_get_lower_bound(&s->temp_end, "U"); + ret = applesmc_get_lower_bound(smc, &s->temp_end, "U"); if (ret) return ret; s->temp_count = s->temp_end - s->temp_begin; - ret = applesmc_init_index(s); + ret = applesmc_init_index(smc, s); if (ret) return ret; - ret = applesmc_has_key(LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); + ret = applesmc_has_key(smc, LIGHT_SENSOR_LEFT_KEY, &left_light_sensor); if (ret) return ret; - ret = applesmc_has_key(LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); + ret = applesmc_has_key(smc, LIGHT_SENSOR_RIGHT_KEY, &right_light_sensor); if (ret) return ret; - ret = applesmc_has_key(MOTION_SENSOR_KEY, &s->has_accelerometer); + ret = applesmc_has_key(smc, MOTION_SENSOR_KEY, &s->has_accelerometer); if (ret) return ret; - ret = applesmc_has_key(BACKLIGHT_KEY, &s->has_key_backlight); + ret = applesmc_has_key(smc, BACKLIGHT_KEY, &s->has_key_backlight); if (ret) return ret; @@ -634,13 +896,13 @@ static int applesmc_init_smcreg_try(void) return 0; } -static void applesmc_destroy_smcreg(void) +static void applesmc_destroy_smcreg(struct applesmc_device *smc) { - kfree(smcreg.index); - smcreg.index = NULL; - kfree(smcreg.cache); - smcreg.cache = NULL; - smcreg.init_complete = false; + kfree(smc->reg.index); + smc->reg.index = NULL; + kfree(smc->reg.cache); + smc->reg.cache = NULL; + smc->reg.init_complete = false; } /* @@ -649,12 +911,12 @@ static void applesmc_destroy_smcreg(void) * Retries until initialization is successful, or the operation times out. * */ -static int applesmc_init_smcreg(void) +static int applesmc_init_smcreg(struct applesmc_device *smc) { int ms, ret; for (ms = 0; ms < INIT_TIMEOUT_MSECS; ms += INIT_WAIT_MSECS) { - ret = applesmc_init_smcreg_try(); + ret = applesmc_init_smcreg_try(smc); if (!ret) { if (ms) pr_info("init_smcreg() took %d ms\n", ms); @@ -663,50 +925,223 @@ static int applesmc_init_smcreg(void) msleep(INIT_WAIT_MSECS); } - applesmc_destroy_smcreg(); + applesmc_destroy_smcreg(smc); return ret; } /* Device model stuff */ -static int applesmc_probe(struct platform_device *dev) + +static int applesmc_init_resources(struct applesmc_device *smc); +static void applesmc_free_resources(struct applesmc_device *smc); +static int applesmc_create_modules(struct applesmc_device *smc); +static void applesmc_destroy_modules(struct applesmc_device *smc); + +static int applesmc_add(struct acpi_device *dev) { + struct applesmc_device *smc; int ret; - ret = applesmc_init_smcreg(); + smc = kzalloc(sizeof(struct applesmc_device), GFP_KERNEL); + if (!smc) + return -ENOMEM; + smc->dev = dev; + smc->ldev = &dev->dev; + mutex_init(&smc->reg.mutex); + + dev_set_drvdata(&dev->dev, smc); + + ret = applesmc_init_resources(smc); if (ret) - return ret; + goto out_mem; + + ret = applesmc_init_smcreg(smc); + if (ret) + goto out_res; + + applesmc_device_init(smc); + + ret = applesmc_create_modules(smc); + if (ret) + goto out_reg; + + return 0; + +out_reg: + applesmc_destroy_smcreg(smc); +out_res: + applesmc_free_resources(smc); +out_mem: + dev_set_drvdata(&dev->dev, NULL); + mutex_destroy(&smc->reg.mutex); + kfree(smc); + + return ret; +} + +static void applesmc_remove(struct acpi_device *dev) +{ + struct applesmc_device *smc = dev_get_drvdata(&dev->dev); + + applesmc_destroy_modules(smc); + applesmc_destroy_smcreg(smc); + applesmc_free_resources(smc); - applesmc_device_init(); + mutex_destroy(&smc->reg.mutex); + kfree(smc); + + return; +} + +static acpi_status applesmc_walk_resources(struct acpi_resource *res, + void *data) +{ + struct applesmc_device *smc = data; + + switch (res->type) { + case ACPI_RESOURCE_TYPE_IO: + if (!smc->port_base_set) { + if (res->data.io.address_length < APPLESMC_NR_PORTS) + return AE_ERROR; + smc->port_base = res->data.io.minimum; + smc->port_base_set = true; + } + return AE_OK; + + case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: + if (!smc->iomem_base_set) { + if (res->data.fixed_memory32.address_length < + APPLESMC_IOMEM_MIN_SIZE) { + dev_warn(smc->ldev, "found iomem but it's too small: %u\n", + res->data.fixed_memory32.address_length); + return AE_OK; + } + smc->iomem_base_addr = res->data.fixed_memory32.address; + smc->iomem_base_size = res->data.fixed_memory32.address_length; + smc->iomem_base_set = true; + } + return AE_OK; + + case ACPI_RESOURCE_TYPE_END_TAG: + if (smc->port_base_set) + return AE_OK; + else + return AE_NOT_FOUND; + + default: + return AE_OK; + } +} + +static int applesmc_try_enable_iomem(struct applesmc_device *smc); + +static int applesmc_init_resources(struct applesmc_device *smc) +{ + int ret; + + ret = acpi_walk_resources(smc->dev->handle, METHOD_NAME__CRS, + applesmc_walk_resources, smc); + if (ACPI_FAILURE(ret)) + return -ENXIO; + + if (!request_region(smc->port_base, APPLESMC_NR_PORTS, "applesmc")) + return -ENXIO; + + if (smc->iomem_base_set) { + if (applesmc_try_enable_iomem(smc)) + smc->iomem_base_set = false; + } + + return 0; +} + +static int applesmc_try_enable_iomem(struct applesmc_device *smc) +{ + u8 test_val, ldkn_version; + + dev_dbg(smc->ldev, "Trying to enable iomem based communication\n"); + smc->iomem_base = ioremap(smc->iomem_base_addr, smc->iomem_base_size); + if (!smc->iomem_base) + goto out; + + /* Apple's driver does this check for some reason */ + test_val = ioread8(smc->iomem_base + APPLESMC_IOMEM_KEY_STATUS); + if (test_val == 0xff) { + dev_warn(smc->ldev, + "iomem enable failed: initial status is 0xff (is %x)\n", + test_val); + goto out_iomem; + } + + if (read_smc(smc, "LDKN", &ldkn_version, 1)) { + dev_warn(smc->ldev, "iomem enable failed: ldkn read failed\n"); + goto out_iomem; + } + + if (ldkn_version < 2) { + dev_warn(smc->ldev, + "iomem enable failed: ldkn version %u is less than minimum (2)\n", + ldkn_version); + goto out_iomem; + } return 0; + +out_iomem: + iounmap(smc->iomem_base); + +out: + return -ENXIO; +} + +static void applesmc_free_resources(struct applesmc_device *smc) +{ + if (smc->iomem_base_set) + iounmap(smc->iomem_base); + release_region(smc->port_base, APPLESMC_NR_PORTS); } /* Synchronize device with memorized backlight state */ static int applesmc_pm_resume(struct device *dev) { - if (smcreg.has_key_backlight) - applesmc_write_key(BACKLIGHT_KEY, backlight_state, 2); + struct applesmc_device *smc = dev_get_drvdata(dev); + + if (smc->reg.has_key_backlight) + applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); + return 0; } /* Reinitialize device on resume from hibernation */ static int applesmc_pm_restore(struct device *dev) { - applesmc_device_init(); + struct applesmc_device *smc = dev_get_drvdata(dev); + + applesmc_device_init(smc); + return applesmc_pm_resume(dev); } +static const struct acpi_device_id applesmc_ids[] = { + {"APP0001", 0}, + {"", 0}, +}; + static const struct dev_pm_ops applesmc_pm_ops = { .resume = applesmc_pm_resume, .restore = applesmc_pm_restore, }; -static struct platform_driver applesmc_driver = { - .probe = applesmc_probe, - .driver = { - .name = "applesmc", - .pm = &applesmc_pm_ops, +static struct acpi_driver applesmc_driver = { + .name = "applesmc", + .class = "applesmc", + .ids = applesmc_ids, + .ops = { + .add = applesmc_add, + .remove = applesmc_remove + }, + .drv = { + .pm = &applesmc_pm_ops }, }; @@ -714,25 +1149,26 @@ static struct platform_driver applesmc_driver = { * applesmc_calibrate - Set our "resting" values. Callers must * hold applesmc_lock. */ -static void applesmc_calibrate(void) +static void applesmc_calibrate(struct applesmc_device *smc) { - applesmc_read_s16(MOTION_SENSOR_X_KEY, &rest_x); - applesmc_read_s16(MOTION_SENSOR_Y_KEY, &rest_y); - rest_x = -rest_x; + applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &smc->rest_x); + applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &smc->rest_y); + smc->rest_x = -smc->rest_x; } static void applesmc_idev_poll(struct input_dev *idev) { + struct applesmc_device *smc = dev_get_drvdata(&idev->dev); s16 x, y; - if (applesmc_read_s16(MOTION_SENSOR_X_KEY, &x)) + if (applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x)) return; - if (applesmc_read_s16(MOTION_SENSOR_Y_KEY, &y)) + if (applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y)) return; x = -x; - input_report_abs(idev, ABS_X, x - rest_x); - input_report_abs(idev, ABS_Y, y - rest_y); + input_report_abs(idev, ABS_X, x - smc->rest_x); + input_report_abs(idev, ABS_Y, y - smc->rest_y); input_sync(idev); } @@ -747,16 +1183,17 @@ static ssize_t applesmc_name_show(struct device *dev, static ssize_t applesmc_position_show(struct device *dev, struct device_attribute *attr, char *buf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; s16 x, y, z; - ret = applesmc_read_s16(MOTION_SENSOR_X_KEY, &x); + ret = applesmc_read_s16(smc, MOTION_SENSOR_X_KEY, &x); if (ret) goto out; - ret = applesmc_read_s16(MOTION_SENSOR_Y_KEY, &y); + ret = applesmc_read_s16(smc, MOTION_SENSOR_Y_KEY, &y); if (ret) goto out; - ret = applesmc_read_s16(MOTION_SENSOR_Z_KEY, &z); + ret = applesmc_read_s16(smc, MOTION_SENSOR_Z_KEY, &z); if (ret) goto out; @@ -770,6 +1207,7 @@ static ssize_t applesmc_position_show(struct device *dev, static ssize_t applesmc_light_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; static int data_length; int ret; @@ -777,7 +1215,7 @@ static ssize_t applesmc_light_show(struct device *dev, u8 buffer[10]; if (!data_length) { - entry = applesmc_get_entry_by_key(LIGHT_SENSOR_LEFT_KEY); + entry = applesmc_get_entry_by_key(smc, LIGHT_SENSOR_LEFT_KEY); if (IS_ERR(entry)) return PTR_ERR(entry); if (entry->len > 10) @@ -786,7 +1224,7 @@ static ssize_t applesmc_light_show(struct device *dev, pr_info("light sensor data length set to %d\n", data_length); } - ret = applesmc_read_key(LIGHT_SENSOR_LEFT_KEY, buffer, data_length); + ret = applesmc_read_key(smc, LIGHT_SENSOR_LEFT_KEY, buffer, data_length); if (ret) goto out; /* newer macbooks report a single 10-bit bigendian value */ @@ -796,7 +1234,7 @@ static ssize_t applesmc_light_show(struct device *dev, } left = buffer[2]; - ret = applesmc_read_key(LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); + ret = applesmc_read_key(smc, LIGHT_SENSOR_RIGHT_KEY, buffer, data_length); if (ret) goto out; right = buffer[2]; @@ -812,7 +1250,8 @@ static ssize_t applesmc_light_show(struct device *dev, static ssize_t applesmc_show_sensor_label(struct device *dev, struct device_attribute *devattr, char *sysfsbuf) { - const char *key = smcreg.index[to_index(devattr)]; + struct applesmc_device *smc = dev_get_drvdata(dev); + const char *key = smc->reg.index[to_index(devattr)]; return sysfs_emit(sysfsbuf, "%s\n", key); } @@ -821,12 +1260,13 @@ static ssize_t applesmc_show_sensor_label(struct device *dev, static ssize_t applesmc_show_temperature(struct device *dev, struct device_attribute *devattr, char *sysfsbuf) { - const char *key = smcreg.index[to_index(devattr)]; + struct applesmc_device *smc = dev_get_drvdata(dev); + const char *key = smc->reg.index[to_index(devattr)]; int ret; s16 value; int temp; - ret = applesmc_read_s16(key, &value); + ret = applesmc_read_s16(smc, key, &value); if (ret) return ret; @@ -838,6 +1278,8 @@ static ssize_t applesmc_show_temperature(struct device *dev, static ssize_t applesmc_show_fan_speed(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; int ret; unsigned int speed = 0; char newkey[5]; @@ -846,11 +1288,21 @@ static ssize_t applesmc_show_fan_speed(struct device *dev, scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], to_index(attr)); - ret = applesmc_read_key(newkey, buffer, 2); + entry = applesmc_get_entry_by_key(smc, newkey); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + if (!strcmp(entry->type, FLOAT_TYPE)) { + ret = applesmc_read_entry(smc, entry, (u8 *) &speed, 4); + speed = applesmc_float_to_u32(speed); + } else { + ret = applesmc_read_entry(smc, entry, buffer, 2); + speed = ((buffer[0] << 8 | buffer[1]) >> 2); + } + if (ret) return ret; - speed = ((buffer[0] << 8 | buffer[1]) >> 2); return sysfs_emit(sysfsbuf, "%u\n", speed); } @@ -858,6 +1310,8 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { + struct applesmc_device *smc = dev_get_drvdata(dev); + const struct applesmc_entry *entry; int ret; unsigned long speed; char newkey[5]; @@ -869,9 +1323,18 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, scnprintf(newkey, sizeof(newkey), fan_speed_fmt[to_option(attr)], to_index(attr)); - buffer[0] = (speed >> 6) & 0xff; - buffer[1] = (speed << 2) & 0xff; - ret = applesmc_write_key(newkey, buffer, 2); + entry = applesmc_get_entry_by_key(smc, newkey); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + if (!strcmp(entry->type, FLOAT_TYPE)) { + speed = applesmc_u32_to_float(speed); + ret = applesmc_write_entry(smc, entry, (u8 *) &speed, 4); + } else { + buffer[0] = (speed >> 6) & 0xff; + buffer[1] = (speed << 2) & 0xff; + ret = applesmc_write_key(smc, newkey, buffer, 2); + } if (ret) return ret; @@ -882,15 +1345,30 @@ static ssize_t applesmc_store_fan_speed(struct device *dev, static ssize_t applesmc_show_fan_manual(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; u16 manual = 0; u8 buffer[2]; + char newkey[5]; + bool has_newkey = false; + + scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); + + ret = applesmc_has_key(smc, newkey, &has_newkey); + if (ret) + return ret; + + if (has_newkey) { + ret = applesmc_read_key(smc, newkey, buffer, 1); + manual = buffer[0]; + } else { + ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); + manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; + } - ret = applesmc_read_key(FANS_MANUAL, buffer, 2); if (ret) return ret; - manual = ((buffer[0] << 8 | buffer[1]) >> to_index(attr)) & 0x01; return sysfs_emit(sysfsbuf, "%d\n", manual); } @@ -898,29 +1376,42 @@ static ssize_t applesmc_store_fan_manual(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; u8 buffer[2]; + char newkey[5]; + bool has_newkey = false; unsigned long input; u16 val; if (kstrtoul(sysfsbuf, 10, &input) < 0) return -EINVAL; - ret = applesmc_read_key(FANS_MANUAL, buffer, 2); + scnprintf(newkey, sizeof(newkey), FAN_MANUAL_FMT, to_index(attr)); + + ret = applesmc_has_key(smc, newkey, &has_newkey); if (ret) - goto out; + return ret; - val = (buffer[0] << 8 | buffer[1]); + if (has_newkey) { + buffer[0] = input & 1; + ret = applesmc_write_key(smc, newkey, buffer, 1); + } else { + ret = applesmc_read_key(smc, FANS_MANUAL, buffer, 2); + val = (buffer[0] << 8 | buffer[1]); + if (ret) + goto out; - if (input) - val = val | (0x01 << to_index(attr)); - else - val = val & ~(0x01 << to_index(attr)); + if (input) + val = val | (0x01 << to_index(attr)); + else + val = val & ~(0x01 << to_index(attr)); - buffer[0] = (val >> 8) & 0xFF; - buffer[1] = val & 0xFF; + buffer[0] = (val >> 8) & 0xFF; + buffer[1] = val & 0xFF; - ret = applesmc_write_key(FANS_MANUAL, buffer, 2); + ret = applesmc_write_key(smc, FANS_MANUAL, buffer, 2); + } out: if (ret) @@ -932,13 +1423,14 @@ static ssize_t applesmc_store_fan_manual(struct device *dev, static ssize_t applesmc_show_fan_position(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; char newkey[5]; u8 buffer[17]; scnprintf(newkey, sizeof(newkey), FAN_ID_FMT, to_index(attr)); - ret = applesmc_read_key(newkey, buffer, 16); + ret = applesmc_read_key(smc, newkey, buffer, 16); buffer[16] = 0; if (ret) @@ -950,43 +1442,79 @@ static ssize_t applesmc_show_fan_position(struct device *dev, static ssize_t applesmc_calibrate_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { - return sysfs_emit(sysfsbuf, "(%d,%d)\n", rest_x, rest_y); + struct applesmc_device *smc = dev_get_drvdata(dev); + + return sysfs_emit(sysfsbuf, "(%d,%d)\n", smc->rest_x, smc->rest_y); } static ssize_t applesmc_calibrate_store(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { - applesmc_calibrate(); + struct applesmc_device *smc = dev_get_drvdata(dev); + + applesmc_calibrate(smc); return count; } static void applesmc_backlight_set(struct work_struct *work) { - applesmc_write_key(BACKLIGHT_KEY, backlight_state, 2); + struct applesmc_device *smc = container_of(work, struct applesmc_device, backlight_work); + + applesmc_write_key(smc, BACKLIGHT_KEY, smc->backlight_state, 2); } -static DECLARE_WORK(backlight_work, &applesmc_backlight_set); static void applesmc_brightness_set(struct led_classdev *led_cdev, enum led_brightness value) { + struct applesmc_device *smc = dev_get_drvdata(led_cdev->dev); int ret; - backlight_state[0] = value; - ret = queue_work(applesmc_led_wq, &backlight_work); + smc->backlight_state[0] = value; + ret = queue_work(smc->backlight_wq, &smc->backlight_work); if (debug && (!ret)) dev_dbg(led_cdev->dev, "work was already on the queue.\n"); } +static ssize_t applesmc_BCLM_store(struct device *dev, + struct device_attribute *attr, char *sysfsbuf, size_t count) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + u8 val; + + if (kstrtou8(sysfsbuf, 10, &val) < 0) + return -EINVAL; + + if (val < 0 || val > 100) + return -EINVAL; + + if (applesmc_write_key(smc, "BCLM", &val, 1)) + return -ENODEV; + return count; +} + +static ssize_t applesmc_BCLM_show(struct device *dev, + struct device_attribute *attr, char *sysfsbuf) +{ + struct applesmc_device *smc = dev_get_drvdata(dev); + u8 val; + + if (applesmc_read_key(smc, "BCLM", &val, 1)) + return -ENODEV; + + return sysfs_emit(sysfsbuf, "%d\n", val); +} + static ssize_t applesmc_key_count_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); int ret; u8 buffer[4]; u32 count; - ret = applesmc_read_key(KEY_COUNT_KEY, buffer, 4); + ret = applesmc_read_key(smc, KEY_COUNT_KEY, buffer, 4); if (ret) return ret; @@ -998,13 +1526,14 @@ static ssize_t applesmc_key_count_show(struct device *dev, static ssize_t applesmc_key_at_index_read_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; int ret; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); - ret = applesmc_read_entry(entry, sysfsbuf, entry->len); + ret = applesmc_read_entry(smc, entry, sysfsbuf, entry->len); if (ret) return ret; @@ -1014,9 +1543,10 @@ static ssize_t applesmc_key_at_index_read_show(struct device *dev, static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); @@ -1026,9 +1556,10 @@ static ssize_t applesmc_key_at_index_data_length_show(struct device *dev, static ssize_t applesmc_key_at_index_type_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); @@ -1038,9 +1569,10 @@ static ssize_t applesmc_key_at_index_type_show(struct device *dev, static ssize_t applesmc_key_at_index_name_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { + struct applesmc_device *smc = dev_get_drvdata(dev); const struct applesmc_entry *entry; - entry = applesmc_get_entry_by_index(key_at_index); + entry = applesmc_get_entry_by_index(smc, smc->key_at_index); if (IS_ERR(entry)) return PTR_ERR(entry); @@ -1050,28 +1582,25 @@ static ssize_t applesmc_key_at_index_name_show(struct device *dev, static ssize_t applesmc_key_at_index_show(struct device *dev, struct device_attribute *attr, char *sysfsbuf) { - return sysfs_emit(sysfsbuf, "%d\n", key_at_index); + struct applesmc_device *smc = dev_get_drvdata(dev); + + return sysfs_emit(sysfsbuf, "%d\n", smc->key_at_index); } static ssize_t applesmc_key_at_index_store(struct device *dev, struct device_attribute *attr, const char *sysfsbuf, size_t count) { + struct applesmc_device *smc = dev_get_drvdata(dev); unsigned long newkey; if (kstrtoul(sysfsbuf, 10, &newkey) < 0 - || newkey >= smcreg.key_count) + || newkey >= smc->reg.key_count) return -EINVAL; - key_at_index = newkey; + smc->key_at_index = newkey; return count; } -static struct led_classdev applesmc_backlight = { - .name = "smc::kbd_backlight", - .default_trigger = "nand-disk", - .brightness_set = applesmc_brightness_set, -}; - static struct applesmc_node_group info_group[] = { { "name", applesmc_name_show }, { "key_count", applesmc_key_count_show }, @@ -1111,19 +1640,25 @@ static struct applesmc_node_group temp_group[] = { { } }; +static struct applesmc_node_group BCLM_group[] = { + { "battery_charge_limit", applesmc_BCLM_show, applesmc_BCLM_store }, + { } +}; + /* Module stuff */ /* * applesmc_destroy_nodes - remove files and free associated memory */ -static void applesmc_destroy_nodes(struct applesmc_node_group *groups) +static void applesmc_destroy_nodes(struct applesmc_device *smc, + struct applesmc_node_group *groups) { struct applesmc_node_group *grp; struct applesmc_dev_attr *node; for (grp = groups; grp->nodes; grp++) { for (node = grp->nodes; node->sda.dev_attr.attr.name; node++) - sysfs_remove_file(&pdev->dev.kobj, + sysfs_remove_file(&smc->dev->dev.kobj, &node->sda.dev_attr.attr); kfree(grp->nodes); grp->nodes = NULL; @@ -1133,7 +1668,8 @@ static void applesmc_destroy_nodes(struct applesmc_node_group *groups) /* * applesmc_create_nodes - create a two-dimensional group of sysfs files */ -static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) +static int applesmc_create_nodes(struct applesmc_device *smc, + struct applesmc_node_group *groups, int num) { struct applesmc_node_group *grp; struct applesmc_dev_attr *node; @@ -1157,7 +1693,7 @@ static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) sysfs_attr_init(attr); attr->name = node->name; attr->mode = 0444 | (grp->store ? 0200 : 0); - ret = sysfs_create_file(&pdev->dev.kobj, attr); + ret = sysfs_create_file(&smc->dev->dev.kobj, attr); if (ret) { attr->name = NULL; goto out; @@ -1167,57 +1703,56 @@ static int applesmc_create_nodes(struct applesmc_node_group *groups, int num) return 0; out: - applesmc_destroy_nodes(groups); + applesmc_destroy_nodes(smc, groups); return ret; } /* Create accelerometer resources */ -static int applesmc_create_accelerometer(void) +static int applesmc_create_accelerometer(struct applesmc_device *smc) { int ret; - - if (!smcreg.has_accelerometer) + if (!smc->reg.has_accelerometer) return 0; - ret = applesmc_create_nodes(accelerometer_group, 1); + ret = applesmc_create_nodes(smc, accelerometer_group, 1); if (ret) goto out; - applesmc_idev = input_allocate_device(); - if (!applesmc_idev) { + smc->idev = input_allocate_device(); + if (!smc->idev) { ret = -ENOMEM; goto out_sysfs; } /* initial calibrate for the input device */ - applesmc_calibrate(); + applesmc_calibrate(smc); /* initialize the input device */ - applesmc_idev->name = "applesmc"; - applesmc_idev->id.bustype = BUS_HOST; - applesmc_idev->dev.parent = &pdev->dev; - input_set_abs_params(applesmc_idev, ABS_X, + smc->idev->name = "applesmc"; + smc->idev->id.bustype = BUS_HOST; + smc->idev->dev.parent = &smc->dev->dev; + input_set_abs_params(smc->idev, ABS_X, -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); - input_set_abs_params(applesmc_idev, ABS_Y, + input_set_abs_params(smc->idev, ABS_Y, -256, 256, APPLESMC_INPUT_FUZZ, APPLESMC_INPUT_FLAT); - ret = input_setup_polling(applesmc_idev, applesmc_idev_poll); + ret = input_setup_polling(smc->idev, applesmc_idev_poll); if (ret) goto out_idev; - input_set_poll_interval(applesmc_idev, APPLESMC_POLL_INTERVAL); + input_set_poll_interval(smc->idev, APPLESMC_POLL_INTERVAL); - ret = input_register_device(applesmc_idev); + ret = input_register_device(smc->idev); if (ret) goto out_idev; return 0; out_idev: - input_free_device(applesmc_idev); + input_free_device(smc->idev); out_sysfs: - applesmc_destroy_nodes(accelerometer_group); + applesmc_destroy_nodes(smc, accelerometer_group); out: pr_warn("driver init failed (ret=%d)!\n", ret); @@ -1225,44 +1760,55 @@ static int applesmc_create_accelerometer(void) } /* Release all resources used by the accelerometer */ -static void applesmc_release_accelerometer(void) +static void applesmc_release_accelerometer(struct applesmc_device *smc) { - if (!smcreg.has_accelerometer) + if (!smc->reg.has_accelerometer) return; - input_unregister_device(applesmc_idev); - applesmc_destroy_nodes(accelerometer_group); + input_unregister_device(smc->idev); + applesmc_destroy_nodes(smc, accelerometer_group); } -static int applesmc_create_light_sensor(void) +static int applesmc_create_light_sensor(struct applesmc_device *smc) { - if (!smcreg.num_light_sensors) + if (!smc->reg.num_light_sensors) return 0; - return applesmc_create_nodes(light_sensor_group, 1); + return applesmc_create_nodes(smc, light_sensor_group, 1); } -static void applesmc_release_light_sensor(void) +static void applesmc_release_light_sensor(struct applesmc_device *smc) { - if (!smcreg.num_light_sensors) + if (!smc->reg.num_light_sensors) return; - applesmc_destroy_nodes(light_sensor_group); + applesmc_destroy_nodes(smc, light_sensor_group); } -static int applesmc_create_key_backlight(void) +static int applesmc_create_key_backlight(struct applesmc_device *smc) { - if (!smcreg.has_key_backlight) + int ret; + + if (!smc->reg.has_key_backlight) return 0; - applesmc_led_wq = create_singlethread_workqueue("applesmc-led"); - if (!applesmc_led_wq) + smc->backlight_wq = create_singlethread_workqueue("applesmc-led"); + if (!smc->backlight_wq) return -ENOMEM; - return led_classdev_register(&pdev->dev, &applesmc_backlight); + + INIT_WORK(&smc->backlight_work, applesmc_backlight_set); + smc->backlight_dev.name = "smc::kbd_backlight"; + smc->backlight_dev.default_trigger = "nand-disk"; + smc->backlight_dev.brightness_set = applesmc_brightness_set; + ret = led_classdev_register(&smc->dev->dev, &smc->backlight_dev); + if (ret) + destroy_workqueue(smc->backlight_wq); + + return ret; } -static void applesmc_release_key_backlight(void) +static void applesmc_release_key_backlight(struct applesmc_device *smc) { - if (!smcreg.has_key_backlight) + if (!smc->reg.has_key_backlight) return; - led_classdev_unregister(&applesmc_backlight); - destroy_workqueue(applesmc_led_wq); + led_classdev_unregister(&smc->backlight_dev); + destroy_workqueue(smc->backlight_wq); } static int applesmc_dmi_match(const struct dmi_system_id *id) @@ -1291,6 +1837,10 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = { DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), DMI_MATCH(DMI_PRODUCT_NAME, "Macmini") }, }, + { applesmc_dmi_match, "Apple iMacPro", { + DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), + DMI_MATCH(DMI_PRODUCT_NAME, "iMacPro") }, + }, { applesmc_dmi_match, "Apple MacPro", { DMI_MATCH(DMI_BOARD_VENDOR, "Apple"), DMI_MATCH(DMI_PRODUCT_NAME, "MacPro") }, @@ -1306,90 +1856,91 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = { { .ident = NULL } }; -static int __init applesmc_init(void) +static int applesmc_create_modules(struct applesmc_device *smc) { int ret; - if (!dmi_check_system(applesmc_whitelist)) { - pr_warn("supported laptop not found!\n"); - ret = -ENODEV; - goto out; - } - - if (!request_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS, - "applesmc")) { - ret = -ENXIO; - goto out; - } - - ret = platform_driver_register(&applesmc_driver); - if (ret) - goto out_region; - - pdev = platform_device_register_simple("applesmc", APPLESMC_DATA_PORT, - NULL, 0); - if (IS_ERR(pdev)) { - ret = PTR_ERR(pdev); - goto out_driver; - } - - /* create register cache */ - ret = applesmc_init_smcreg(); + ret = applesmc_create_nodes(smc, info_group, 1); if (ret) - goto out_device; - - ret = applesmc_create_nodes(info_group, 1); + goto out; + ret = applesmc_create_nodes(smc, BCLM_group, 1); if (ret) - goto out_smcreg; + goto out_info; - ret = applesmc_create_nodes(fan_group, smcreg.fan_count); + ret = applesmc_create_nodes(smc, fan_group, smc->reg.fan_count); if (ret) - goto out_info; + goto out_bclm; - ret = applesmc_create_nodes(temp_group, smcreg.index_count); + ret = applesmc_create_nodes(smc, temp_group, smc->reg.index_count); if (ret) goto out_fans; - ret = applesmc_create_accelerometer(); + ret = applesmc_create_accelerometer(smc); if (ret) goto out_temperature; - ret = applesmc_create_light_sensor(); + ret = applesmc_create_light_sensor(smc); if (ret) goto out_accelerometer; - ret = applesmc_create_key_backlight(); + ret = applesmc_create_key_backlight(smc); if (ret) goto out_light_sysfs; - hwmon_dev = hwmon_device_register(&pdev->dev); - if (IS_ERR(hwmon_dev)) { - ret = PTR_ERR(hwmon_dev); + smc->hwmon_dev = hwmon_device_register(&smc->dev->dev); + if (IS_ERR(smc->hwmon_dev)) { + ret = PTR_ERR(smc->hwmon_dev); goto out_light_ledclass; } return 0; out_light_ledclass: - applesmc_release_key_backlight(); + applesmc_release_key_backlight(smc); out_light_sysfs: - applesmc_release_light_sensor(); + applesmc_release_light_sensor(smc); out_accelerometer: - applesmc_release_accelerometer(); + applesmc_release_accelerometer(smc); out_temperature: - applesmc_destroy_nodes(temp_group); + applesmc_destroy_nodes(smc, temp_group); out_fans: - applesmc_destroy_nodes(fan_group); + applesmc_destroy_nodes(smc, fan_group); +out_bclm: + applesmc_destroy_nodes(smc, BCLM_group); out_info: - applesmc_destroy_nodes(info_group); -out_smcreg: - applesmc_destroy_smcreg(); -out_device: - platform_device_unregister(pdev); -out_driver: - platform_driver_unregister(&applesmc_driver); -out_region: - release_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS); + applesmc_destroy_nodes(smc, info_group); +out: + return ret; +} + +static void applesmc_destroy_modules(struct applesmc_device *smc) +{ + hwmon_device_unregister(smc->hwmon_dev); + applesmc_release_key_backlight(smc); + applesmc_release_light_sensor(smc); + applesmc_release_accelerometer(smc); + applesmc_destroy_nodes(smc, temp_group); + applesmc_destroy_nodes(smc, fan_group); + applesmc_destroy_nodes(smc, BCLM_group); + applesmc_destroy_nodes(smc, info_group); +} + +static int __init applesmc_init(void) +{ + int ret; + + if (!dmi_check_system(applesmc_whitelist)) { + pr_warn("supported laptop not found!\n"); + ret = -ENODEV; + goto out; + } + + ret = acpi_bus_register_driver(&applesmc_driver); + if (ret) + goto out; + + return 0; + out: pr_warn("driver init failed (ret=%d)!\n", ret); return ret; @@ -1397,23 +1948,14 @@ static int __init applesmc_init(void) static void __exit applesmc_exit(void) { - hwmon_device_unregister(hwmon_dev); - applesmc_release_key_backlight(); - applesmc_release_light_sensor(); - applesmc_release_accelerometer(); - applesmc_destroy_nodes(temp_group); - applesmc_destroy_nodes(fan_group); - applesmc_destroy_nodes(info_group); - applesmc_destroy_smcreg(); - platform_device_unregister(pdev); - platform_driver_unregister(&applesmc_driver); - release_region(APPLESMC_DATA_PORT, APPLESMC_NR_PORTS); + acpi_bus_unregister_driver(&applesmc_driver); } module_init(applesmc_init); module_exit(applesmc_exit); MODULE_AUTHOR("Nicolas Boichat"); +MODULE_AUTHOR("Paul Pawlowski"); MODULE_DESCRIPTION("Apple SMC"); MODULE_LICENSE("GPL v2"); MODULE_DEVICE_TABLE(dmi, applesmc_whitelist); diff --git a/drivers/pci/vgaarb.c b/drivers/pci/vgaarb.c index 78748e8d2dba..2b2b558cebe6 100644 --- a/drivers/pci/vgaarb.c +++ b/drivers/pci/vgaarb.c @@ -143,6 +143,7 @@ void vga_set_default_device(struct pci_dev *pdev) pci_dev_put(vga_default); vga_default = pci_dev_get(pdev); } +EXPORT_SYMBOL_GPL(vga_set_default_device); /** * vga_remove_vgacon - deactivate VGA console diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c index 1417e230edbd..e69785af8e1d 100644 --- a/drivers/platform/x86/apple-gmux.c +++ b/drivers/platform/x86/apple-gmux.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,10 @@ struct apple_gmux_config { # define MMIO_GMUX_MAX_BRIGHTNESS 0xffff +static bool force_igd; +module_param(force_igd, bool, 0); +MODULE_PARM_DESC(force_idg, "Switch gpu to igd on module load. Make sure that you have apple-set-os set up and the iGPU is in `lspci -s 00:02.0`. (default: false) (bool)"); + static u8 gmux_pio_read8(struct apple_gmux_data *gmux_data, int port) { return inb(gmux_data->iostart + port); @@ -945,6 +950,19 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) gmux_enable_interrupts(gmux_data); gmux_read_switch_state(gmux_data); + if (force_igd) { + struct pci_dev *pdev; + + pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(2, 0)); + if (pdev) { + pr_info("Switching to IGD"); + gmux_switchto(VGA_SWITCHEROO_IGD); + vga_set_default_device(pdev); + } else { + pr_err("force_idg is true, but couldn't find iGPU at 00:02.0! Is apple-set-os working?"); + } + } + /* * Retina MacBook Pros cannot switch the panel's AUX separately * and need eDP pre-calibration. They are distinguishable from diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 6388cbe1e56b..50f092732796 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,16 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + config APPLE_MAILBOX tristate "Apple SoC mailboxes" depends on PM @@ -30,6 +40,20 @@ config APPLE_RTKIT Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 4d9ab8f3037b..5e526a9edcf2 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,10 +1,16 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c new file mode 100644 index 000000000000..3a0d7964007c --- /dev/null +++ b/drivers/soc/apple/dockchannel.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + struct device_node *child; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq <= 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + for_each_child_of_node(dev->of_node, child) + of_platform_device_create(child, NULL, dev); + + return 0; +} + +static void dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c new file mode 100644 index 000000000000..080d083ed9bd --- /dev/null +++ b/drivers/soc/apple/rtkit-helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static void apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index 075e775d3868..e1cc0d60eeb6 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -50,4 +50,6 @@ source "drivers/staging/vme_user/Kconfig" source "drivers/staging/gpib/Kconfig" +source "drivers/staging/apple-bce/Kconfig" + endif # STAGING diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index e681e403509c..4045c588b3b4 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -14,3 +14,4 @@ obj-$(CONFIG_GREYBUS) += greybus/ obj-$(CONFIG_BCM2835_VCHIQ) += vc04_services/ obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ obj-$(CONFIG_GPIB) += gpib/ +obj-$(CONFIG_APPLE_BCE) += apple-bce/ diff --git a/drivers/staging/apple-bce/Kconfig b/drivers/staging/apple-bce/Kconfig new file mode 100644 index 000000000000..fe92bc441e89 --- /dev/null +++ b/drivers/staging/apple-bce/Kconfig @@ -0,0 +1,18 @@ +config APPLE_BCE + tristate "Apple BCE driver (VHCI and Audio support)" + default m + depends on X86 + select SOUND + select SND + select SND_PCM + select SND_JACK + help + VHCI and audio support on Apple MacBooks with the T2 Chip. + This driver is divided in three components: + - BCE (Buffer Copy Engine): which establishes a basic communication + channel with the T2 chip. This component is required by the other two: + - VHCI (Virtual Host Controller Interface): Access to keyboard, mouse + and other system devices depend on this virtual USB host controller + - Audio: a driver for the T2 audio interface. + + If "M" is selected, the module will be called apple-bce.' diff --git a/drivers/staging/apple-bce/Makefile b/drivers/staging/apple-bce/Makefile new file mode 100644 index 000000000000..8cfbd3f64af6 --- /dev/null +++ b/drivers/staging/apple-bce/Makefile @@ -0,0 +1,28 @@ +modname := apple-bce +obj-$(CONFIG_APPLE_BCE) += $(modname).o + +apple-bce-objs := apple_bce.o mailbox.o queue.o queue_dma.o vhci/vhci.o vhci/queue.o vhci/transfer.o audio/audio.o audio/protocol.o audio/protocol_bce.o audio/pcm.o + +MY_CFLAGS += -DWITHOUT_NVME_PATCH +#MY_CFLAGS += -g -DDEBUG +ccflags-y += ${MY_CFLAGS} +CC += ${MY_CFLAGS} + +KVERSION := $(KERNELRELEASE) +ifeq ($(origin KERNELRELEASE), undefined) +KVERSION := $(shell uname -r) +endif + +KDIR := /lib/modules/$(KVERSION)/build +PWD := $(shell pwd) + +.PHONY: all + +all: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean + +install: + $(MAKE) -C $(KDIR) M=$(PWD) modules_install diff --git a/drivers/staging/apple-bce/apple_bce.c b/drivers/staging/apple-bce/apple_bce.c new file mode 100644 index 000000000000..4fd2415d7028 --- /dev/null +++ b/drivers/staging/apple-bce/apple_bce.c @@ -0,0 +1,445 @@ +#include "apple_bce.h" +#include +#include +#include "audio/audio.h" +#include + +static dev_t bce_chrdev; +static struct class *bce_class; + +struct apple_bce_device *global_bce; + +static int bce_create_command_queues(struct apple_bce_device *bce); +static void bce_free_command_queues(struct apple_bce_device *bce); +static irqreturn_t bce_handle_mb_irq(int irq, void *dev); +static irqreturn_t bce_handle_dma_irq(int irq, void *dev); +static int bce_fw_version_handshake(struct apple_bce_device *bce); +static int bce_register_command_queue(struct apple_bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq); + +static int apple_bce_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct apple_bce_device *bce = NULL; + int status = 0; + int nvec; + + pr_info("apple-bce: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "apple-bce")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + nvec = pci_alloc_irq_vectors(dev, 1, 8, PCI_IRQ_MSI); + if (nvec < 5) { + status = -EINVAL; + goto fail; + } + + bce = kzalloc(sizeof(struct apple_bce_device), GFP_KERNEL); + if (!bce) { + status = -ENOMEM; + goto fail; + } + + bce->pci = dev; + pci_set_drvdata(dev, bce); + + bce->devt = bce_chrdev; + bce->dev = device_create(bce_class, &dev->dev, bce->devt, NULL, "apple-bce"); + if (IS_ERR_OR_NULL(bce->dev)) { + status = PTR_ERR(bce_class); + goto fail; + } + + bce->reg_mem_mb = pci_iomap(dev, 4, 0); + bce->reg_mem_dma = pci_iomap(dev, 2, 0); + + if (IS_ERR_OR_NULL(bce->reg_mem_mb) || IS_ERR_OR_NULL(bce->reg_mem_dma)) { + dev_warn(&dev->dev, "apple-bce: Failed to pci_iomap required regions\n"); + goto fail; + } + + bce_mailbox_init(&bce->mbox, bce->reg_mem_mb); + bce_timestamp_init(&bce->timestamp, bce->reg_mem_mb); + + spin_lock_init(&bce->queues_lock); + ida_init(&bce->queue_ida); + + if ((status = pci_request_irq(dev, 0, bce_handle_mb_irq, NULL, dev, "bce_mbox"))) + goto fail; + if ((status = pci_request_irq(dev, 4, NULL, bce_handle_dma_irq, dev, "bce_dma"))) + goto fail_interrupt_0; + + if ((status = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(37)))) { + dev_warn(&dev->dev, "dma: Setting mask failed\n"); + goto fail_interrupt; + } + + /* Gets the function 0's interface. This is needed because Apple only accepts DMA on our function if function 0 + is a bus master, so we need to work around this. */ + bce->pci0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); +#ifndef WITHOUT_NVME_PATCH + if ((status = pci_enable_device_mem(bce->pci0))) { + dev_warn(&dev->dev, "apple-bce: failed to enable function 0\n"); + goto fail_dev0; + } +#endif + pci_set_master(bce->pci0); + + bce_timestamp_start(&bce->timestamp, true); + + if ((status = bce_fw_version_handshake(bce))) + goto fail_ts; + pr_info("apple-bce: handshake done\n"); + + if ((status = bce_create_command_queues(bce))) { + pr_info("apple-bce: Creating command queues failed\n"); + goto fail_ts; + } + + global_bce = bce; + + bce_vhci_create(bce, &bce->vhci); + + return 0; + +fail_ts: + bce_timestamp_stop(&bce->timestamp); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +fail_dev0: +#endif + pci_dev_put(bce->pci0); +fail_interrupt: + pci_free_irq(dev, 4, dev); +fail_interrupt_0: + pci_free_irq(dev, 0, dev); +fail: + if (bce && bce->dev) { + device_destroy(bce_class, bce->devt); + + if (!IS_ERR_OR_NULL(bce->reg_mem_mb)) + pci_iounmap(dev, bce->reg_mem_mb); + if (!IS_ERR_OR_NULL(bce->reg_mem_dma)) + pci_iounmap(dev, bce->reg_mem_dma); + + kfree(bce); + } + + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + +static int bce_create_command_queues(struct apple_bce_device *bce) +{ + int status; + struct bce_queue_memcfg *cfg; + + bce->cmd_cq = bce_alloc_cq(bce, 0, 0x20); + bce->cmd_cmdq = bce_alloc_cmdq(bce, 1, 0x20); + if (bce->cmd_cq == NULL || bce->cmd_cmdq == NULL) { + status = -ENOMEM; + goto err; + } + bce->queues[0] = (struct bce_queue *) bce->cmd_cq; + bce->queues[1] = (struct bce_queue *) bce->cmd_cmdq->sq; + + cfg = kzalloc(sizeof(struct bce_queue_memcfg), GFP_KERNEL); + if (!cfg) { + status = -ENOMEM; + goto err; + } + bce_get_cq_memcfg(bce->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, false))) + goto err; + bce_get_sq_memcfg(bce->cmd_cmdq->sq, bce->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, true))) + goto err; + kfree(cfg); + + return 0; + +err: + if (bce->cmd_cq) + bce_free_cq(bce, bce->cmd_cq); + if (bce->cmd_cmdq) + bce_free_cmdq(bce, bce->cmd_cmdq); + return status; +} + +static void bce_free_command_queues(struct apple_bce_device *bce) +{ + bce_free_cq(bce, bce->cmd_cq); + bce_free_cmdq(bce, bce->cmd_cmdq); + bce->cmd_cq = NULL; + bce->queues[0] = NULL; +} + +static irqreturn_t bce_handle_mb_irq(int irq, void *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(dev); + bce_mailbox_handle_interrupt(&bce->mbox); + return IRQ_HANDLED; +} + +static irqreturn_t bce_handle_dma_irq(int irq, void *dev) +{ + int i; + struct apple_bce_device *bce = pci_get_drvdata(dev); + spin_lock(&bce->queues_lock); + for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) + if (bce->queues[i] && bce->queues[i]->type == BCE_QUEUE_CQ) + bce_handle_cq_completions(bce, (struct bce_queue_cq *) bce->queues[i]); + spin_unlock(&bce->queues_lock); + return IRQ_HANDLED; +} + +static int bce_fw_version_handshake(struct apple_bce_device *bce) +{ + u64 result; + int status; + + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VERSION, BC_PROTOCOL_VERSION), + &result))) + return status; + if (BCE_MB_TYPE(result) != BCE_MB_SET_FW_PROTOCOL_VERSION || + BCE_MB_VALUE(result) != BC_PROTOCOL_VERSION) { + pr_err("apple-bce: FW version handshake failed %x:%llx\n", BCE_MB_TYPE(result), BCE_MB_VALUE(result)); + return -EINVAL; + } + return 0; +} + +static int bce_register_command_queue(struct apple_bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq) +{ + int status; + int cmd_type; + u64 result; + // OS X uses an bidirectional direction, but that's not really needed + dma_addr_t a = dma_map_single(&bce->pci->dev, cfg, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (dma_mapping_error(&bce->pci->dev, a)) + return -ENOMEM; + cmd_type = is_sq ? BCE_MB_REGISTER_COMMAND_SQ : BCE_MB_REGISTER_COMMAND_CQ; + status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(cmd_type, a), &result); + dma_unmap_single(&bce->pci->dev, a, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (status) + return status; + if (BCE_MB_TYPE(result) != BCE_MB_REGISTER_COMMAND_QUEUE_REPLY) + return -EINVAL; + return 0; +} + +static void apple_bce_remove(struct pci_dev *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(dev); + bce->is_being_removed = true; + + bce_vhci_destroy(&bce->vhci); + + bce_timestamp_stop(&bce->timestamp); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +#endif + pci_dev_put(bce->pci0); + pci_free_irq(dev, 0, dev); + pci_free_irq(dev, 4, dev); + bce_free_command_queues(bce); + pci_iounmap(dev, bce->reg_mem_mb); + pci_iounmap(dev, bce->reg_mem_dma); + device_destroy(bce_class, bce->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + kfree(bce); +} + +static int bce_save_state_and_sleep(struct apple_bce_device *bce) +{ + int attempt, status = 0; + u64 resp; + dma_addr_t dma_addr; + void *dma_ptr = NULL; + size_t size = max(PAGE_SIZE, 4096UL); + + for (attempt = 0; attempt < 5; ++attempt) { + pr_debug("apple-bce: suspend: attempt %i, buffer size %li\n", attempt, size); + dma_ptr = dma_alloc_coherent(&bce->pci->dev, size, &dma_addr, GFP_KERNEL); + if (!dma_ptr) { + pr_err("apple-bce: suspend failed (data alloc failed)\n"); + break; + } + BUG_ON((dma_addr % 4096) != 0); + status = bce_mailbox_send(&bce->mbox, + BCE_MB_MSG(BCE_MB_SAVE_STATE_AND_SLEEP, (dma_addr & ~(4096LLU - 1)) | (size / 4096)), &resp); + if (status) { + pr_err("apple-bce: suspend failed (mailbox send)\n"); + break; + } + if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + bce->saved_data_dma_addr = dma_addr; + bce->saved_data_dma_ptr = dma_ptr; + bce->saved_data_dma_size = size; + return 0; + } else if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_STATE_AND_SLEEP_FAILURE) { + dma_free_coherent(&bce->pci->dev, size, dma_ptr, dma_addr); + /* The 0x10ff magic value was extracted from Apple's driver */ + size = (BCE_MB_VALUE(resp) + 0x10ff) & ~(4096LLU - 1); + pr_debug("apple-bce: suspend: device requested a larger buffer (%li)\n", size); + continue; + } else { + pr_err("apple-bce: suspend failed (invalid device response)\n"); + status = -EINVAL; + break; + } + } + if (dma_ptr) + dma_free_coherent(&bce->pci->dev, size, dma_ptr, dma_addr); + if (!status) + return bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), &resp); + return status; +} + +static int bce_restore_state_and_wake(struct apple_bce_device *bce) +{ + int status; + u64 resp; + if (!bce->saved_data_dma_ptr) { + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &resp))) { + pr_err("apple-bce: resume with no state failed (mailbox send)\n"); + return status; + } + if (BCE_MB_TYPE(resp) != BCE_MB_RESTORE_NO_STATE) { + pr_err("apple-bce: resume with no state failed (invalid device response)\n"); + return -EINVAL; + } + return 0; + } + + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_STATE_AND_WAKE, + (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), &resp))) { + pr_err("apple-bce: resume with state failed (mailbox send)\n"); + goto finish_with_state; + } + if (BCE_MB_TYPE(resp) != BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + pr_err("apple-bce: resume with state failed (invalid device response)\n"); + status = -EINVAL; + goto finish_with_state; + } + +finish_with_state: + dma_free_coherent(&bce->pci->dev, bce->saved_data_dma_size, bce->saved_data_dma_ptr, bce->saved_data_dma_addr); + bce->saved_data_dma_ptr = NULL; + return status; +} + +static int apple_bce_suspend(struct device *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + + bce_timestamp_stop(&bce->timestamp); + + if ((status = bce_save_state_and_sleep(bce))) + return status; + + return 0; +} + +static int apple_bce_resume(struct device *dev) +{ + struct apple_bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + + pci_set_master(bce->pci); + pci_set_master(bce->pci0); + + if ((status = bce_restore_state_and_wake(bce))) + return status; + + bce_timestamp_start(&bce->timestamp, false); + + return 0; +} + +static struct pci_device_id apple_bce_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1801) }, + { 0, }, +}; + +MODULE_DEVICE_TABLE(pci, apple_bce_ids); + +struct dev_pm_ops apple_bce_pci_driver_pm = { + .suspend = apple_bce_suspend, + .resume = apple_bce_resume +}; +struct pci_driver apple_bce_pci_driver = { + .name = "apple-bce", + .id_table = apple_bce_ids, + .probe = apple_bce_probe, + .remove = apple_bce_remove, + .driver = { + .pm = &apple_bce_pci_driver_pm + } +}; + + +static int __init apple_bce_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&bce_chrdev, 0, 1, "apple-bce"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_class = class_create(THIS_MODULE, "apple-bce"); +#else + bce_class = class_create("apple-bce"); +#endif + if (IS_ERR(bce_class)) { + result = PTR_ERR(bce_class); + goto fail_class; + } + if ((result = bce_vhci_module_init())) { + pr_err("apple-bce: bce-vhci init failed"); + goto fail_class; + } + + result = pci_register_driver(&apple_bce_pci_driver); + if (result) + goto fail_drv; + + aaudio_module_init(); + + return 0; + +fail_drv: + pci_unregister_driver(&apple_bce_pci_driver); +fail_class: + class_destroy(bce_class); +fail_chrdev: + unregister_chrdev_region(bce_chrdev, 1); + if (!result) + result = -EINVAL; + return result; +} +static void __exit apple_bce_module_exit(void) +{ + pci_unregister_driver(&apple_bce_pci_driver); + + aaudio_module_exit(); + bce_vhci_module_exit(); + class_destroy(bce_class); + unregister_chrdev_region(bce_chrdev, 1); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("MrARM"); +MODULE_DESCRIPTION("Apple BCE Driver"); +MODULE_VERSION("0.01"); +module_init(apple_bce_module_init); +module_exit(apple_bce_module_exit); diff --git a/drivers/staging/apple-bce/apple_bce.h b/drivers/staging/apple-bce/apple_bce.h new file mode 100644 index 000000000000..f13ab8d5742e --- /dev/null +++ b/drivers/staging/apple-bce/apple_bce.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include "mailbox.h" +#include "queue.h" +#include "vhci/vhci.h" + +#define BC_PROTOCOL_VERSION 0x20001 +#define BCE_MAX_QUEUE_COUNT 0x100 + +#define BCE_QUEUE_USER_MIN 2 +#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) + +struct apple_bce_device { + struct pci_dev *pci, *pci0; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_mb; + void __iomem *reg_mem_dma; + struct bce_mailbox mbox; + struct bce_timestamp timestamp; + struct bce_queue *queues[BCE_MAX_QUEUE_COUNT]; + struct spinlock queues_lock; + struct ida queue_ida; + struct bce_queue_cq *cmd_cq; + struct bce_queue_cmdq *cmd_cmdq; + struct bce_queue_sq *int_sq_list[BCE_MAX_QUEUE_COUNT]; + bool is_being_removed; + + dma_addr_t saved_data_dma_addr; + void *saved_data_dma_ptr; + size_t saved_data_dma_size; + + struct bce_vhci vhci; +}; + +extern struct apple_bce_device *global_bce; \ No newline at end of file diff --git a/drivers/staging/apple-bce/audio/audio.c b/drivers/staging/apple-bce/audio/audio.c new file mode 100644 index 000000000000..bd16ddd16c1d --- /dev/null +++ b/drivers/staging/apple-bce/audio/audio.c @@ -0,0 +1,711 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "audio.h" +#include "pcm.h" +#include + +static int aaudio_alsa_index = SNDRV_DEFAULT_IDX1; +static char *aaudio_alsa_id = SNDRV_DEFAULT_STR1; + +static dev_t aaudio_chrdev; +static struct class *aaudio_class; + +static int aaudio_init_cmd(struct aaudio_device *a); +static int aaudio_init_bs(struct aaudio_device *a); +static void aaudio_init_dev(struct aaudio_device *a, aaudio_device_id_t dev_id); +static void aaudio_free_dev(struct aaudio_subdevice *sdev); + +static int aaudio_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct aaudio_device *aaudio = NULL; + struct aaudio_subdevice *sdev = NULL; + int status = 0; + u32 cfg; + + pr_info("aaudio: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "aaudio")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + + aaudio = kzalloc(sizeof(struct aaudio_device), GFP_KERNEL); + if (!aaudio) { + status = -ENOMEM; + goto fail; + } + + aaudio->bce = global_bce; + if (!aaudio->bce) { + dev_warn(&dev->dev, "aaudio: No BCE available\n"); + status = -EINVAL; + goto fail; + } + + aaudio->pci = dev; + pci_set_drvdata(dev, aaudio); + + aaudio->devt = aaudio_chrdev; + aaudio->dev = device_create(aaudio_class, &dev->dev, aaudio->devt, NULL, "aaudio"); + if (IS_ERR_OR_NULL(aaudio->dev)) { + status = PTR_ERR(aaudio_class); + goto fail; + } + device_link_add(aaudio->dev, aaudio->bce->dev, DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); + + init_completion(&aaudio->remote_alive); + INIT_LIST_HEAD(&aaudio->subdevice_list); + + /* Init: set an unknown flag in the bitset */ + if (pci_read_config_dword(dev, 4, &cfg)) + dev_warn(&dev->dev, "aaudio: pci_read_config_dword fail\n"); + if (pci_write_config_dword(dev, 4, cfg | 6u)) + dev_warn(&dev->dev, "aaudio: pci_write_config_dword fail\n"); + + dev_info(aaudio->dev, "aaudio: bs len = %llx\n", pci_resource_len(dev, 0)); + aaudio->reg_mem_bs_dma = pci_resource_start(dev, 0); + aaudio->reg_mem_bs = pci_iomap(dev, 0, 0); + aaudio->reg_mem_cfg = pci_iomap(dev, 4, 0); + + aaudio->reg_mem_gpr = (u32 __iomem *) ((u8 __iomem *) aaudio->reg_mem_cfg + 0xC000); + + if (IS_ERR_OR_NULL(aaudio->reg_mem_bs) || IS_ERR_OR_NULL(aaudio->reg_mem_cfg)) { + dev_warn(&dev->dev, "aaudio: Failed to pci_iomap required regions\n"); + goto fail; + } + + if (aaudio_bce_init(aaudio)) { + dev_warn(&dev->dev, "aaudio: Failed to init BCE command transport\n"); + goto fail; + } + + if (snd_card_new(aaudio->dev, aaudio_alsa_index, aaudio_alsa_id, THIS_MODULE, 0, &aaudio->card)) { + dev_err(&dev->dev, "aaudio: Failed to create ALSA card\n"); + goto fail; + } + + strcpy(aaudio->card->shortname, "Apple T2 Audio"); + strcpy(aaudio->card->longname, "Apple T2 Audio"); + strcpy(aaudio->card->mixername, "Apple T2 Audio"); + /* Dynamic alsa ids start at 100 */ + aaudio->next_alsa_id = 100; + + if (aaudio_init_cmd(aaudio)) { + dev_err(&dev->dev, "aaudio: Failed to initialize over BCE\n"); + goto fail_snd; + } + + if (aaudio_init_bs(aaudio)) { + dev_err(&dev->dev, "aaudio: Failed to initialize BufferStruct\n"); + goto fail_snd; + } + + if ((status = aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_ON))) { + dev_err(&dev->dev, "Failed to set remote access\n"); + return status; + } + + if (snd_card_register(aaudio->card)) { + dev_err(&dev->dev, "aaudio: Failed to register ALSA sound device\n"); + goto fail_snd; + } + + list_for_each_entry(sdev, &aaudio->subdevice_list, list) { + struct aaudio_buffer_struct_device *dev = &aaudio->bs->devices[sdev->buf_id]; + + if (sdev->out_stream_cnt == 1 && !strcmp(dev->name, "Speaker")) { + struct snd_pcm_hardware *hw = sdev->out_streams[0].alsa_hw_desc; + + snprintf(aaudio->card->driver, sizeof(aaudio->card->driver) / sizeof(char), "AppleT2x%d", hw->channels_min); + } + } + + return 0; + +fail_snd: + snd_card_free(aaudio->card); +fail: + if (aaudio && aaudio->dev) + device_destroy(aaudio_class, aaudio->devt); + kfree(aaudio); + + if (!IS_ERR_OR_NULL(aaudio->reg_mem_bs)) + pci_iounmap(dev, aaudio->reg_mem_bs); + if (!IS_ERR_OR_NULL(aaudio->reg_mem_cfg)) + pci_iounmap(dev, aaudio->reg_mem_cfg); + + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + + + +static void aaudio_remove(struct pci_dev *dev) +{ + struct aaudio_subdevice *sdev; + struct aaudio_device *aaudio = pci_get_drvdata(dev); + + snd_card_free(aaudio->card); + while (!list_empty(&aaudio->subdevice_list)) { + sdev = list_first_entry(&aaudio->subdevice_list, struct aaudio_subdevice, list); + list_del(&sdev->list); + aaudio_free_dev(sdev); + } + pci_iounmap(dev, aaudio->reg_mem_bs); + pci_iounmap(dev, aaudio->reg_mem_cfg); + device_destroy(aaudio_class, aaudio->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + kfree(aaudio); +} + +static int aaudio_suspend(struct device *dev) +{ + struct aaudio_device *aaudio = pci_get_drvdata(to_pci_dev(dev)); + + if (aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_OFF)) + dev_warn(aaudio->dev, "Failed to reset remote access\n"); + + pci_disable_device(aaudio->pci); + return 0; +} + +static int aaudio_resume(struct device *dev) +{ + int status; + struct aaudio_device *aaudio = pci_get_drvdata(to_pci_dev(dev)); + + if ((status = pci_enable_device(aaudio->pci))) + return status; + pci_set_master(aaudio->pci); + + if ((status = aaudio_cmd_set_remote_access(aaudio, AAUDIO_REMOTE_ACCESS_ON))) { + dev_err(aaudio->dev, "Failed to set remote access\n"); + return status; + } + + return 0; +} + +static int aaudio_init_cmd(struct aaudio_device *a) +{ + int status; + struct aaudio_send_ctx sctx; + struct aaudio_msg buf; + u64 dev_cnt, dev_i; + aaudio_device_id_t *dev_l; + + if ((status = aaudio_send(a, &sctx, 500, + aaudio_msg_write_alive_notification, 1, 3))) { + dev_err(a->dev, "Sending alive notification failed\n"); + return status; + } + + if (wait_for_completion_timeout(&a->remote_alive, msecs_to_jiffies(500)) == 0) { + dev_err(a->dev, "Timed out waiting for remote\n"); + return -ETIMEDOUT; + } + dev_info(a->dev, "Continuing init\n"); + + buf = aaudio_reply_alloc(); + if ((status = aaudio_cmd_get_device_list(a, &buf, &dev_l, &dev_cnt))) { + dev_err(a->dev, "Failed to get device list\n"); + aaudio_reply_free(&buf); + return status; + } + for (dev_i = 0; dev_i < dev_cnt; ++dev_i) + aaudio_init_dev(a, dev_l[dev_i]); + aaudio_reply_free(&buf); + + return 0; +} + +static void aaudio_init_stream_info(struct aaudio_subdevice *sdev, struct aaudio_stream *strm); +static void aaudio_handle_jack_connection_change(struct aaudio_subdevice *sdev); + +static void aaudio_init_dev(struct aaudio_device *a, aaudio_device_id_t dev_id) +{ + struct aaudio_subdevice *sdev; + struct aaudio_msg buf = aaudio_reply_alloc(); + u64 uid_len, stream_cnt, i; + aaudio_object_id_t *stream_list; + char *uid; + + sdev = kzalloc(sizeof(struct aaudio_subdevice), GFP_KERNEL); + + if (aaudio_cmd_get_property(a, &buf, dev_id, dev_id, AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_UID, 0), + NULL, 0, (void **) &uid, &uid_len) || uid_len > AAUDIO_DEVICE_MAX_UID_LEN) { + dev_err(a->dev, "Failed to get device uid for device %llx\n", dev_id); + goto fail; + } + dev_info(a->dev, "Remote device %llx %.*s\n", dev_id, (int) uid_len, uid); + + sdev->a = a; + INIT_LIST_HEAD(&sdev->list); + sdev->dev_id = dev_id; + sdev->buf_id = AAUDIO_BUFFER_ID_NONE; + strncpy(sdev->uid, uid, uid_len); + sdev->uid[uid_len + 1] = '\0'; + + if (aaudio_cmd_get_primitive_property(a, dev_id, dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_INPUT, AAUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->in_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device input latency\n"); + if (aaudio_cmd_get_primitive_property(a, dev_id, dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->out_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device output latency\n"); + + if (aaudio_cmd_get_input_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get input stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > AAUDIO_DEIVCE_MAX_INPUT_STREAMS) { + dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, AAUDIO_DEIVCE_MAX_INPUT_STREAMS); + stream_cnt = AAUDIO_DEIVCE_MAX_INPUT_STREAMS; + } + sdev->in_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->in_streams[i].id = stream_list[i]; + sdev->in_streams[i].buffer_cnt = 0; + aaudio_init_stream_info(sdev, &sdev->in_streams[i]); + sdev->in_streams[i].latency += sdev->in_latency; + } + + if (aaudio_cmd_get_output_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get output stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS) { + dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS); + stream_cnt = AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS; + } + sdev->out_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->out_streams[i].id = stream_list[i]; + sdev->out_streams[i].buffer_cnt = 0; + aaudio_init_stream_info(sdev, &sdev->out_streams[i]); + sdev->out_streams[i].latency += sdev->in_latency; + } + + if (sdev->is_pcm) + aaudio_create_pcm(sdev); + /* Headphone Jack status */ + if (!strcmp(sdev->uid, "Codec Output")) { + if (snd_jack_new(a->card, sdev->uid, SND_JACK_HEADPHONE, &sdev->jack, true, false)) + dev_warn(a->dev, "Failed to create an attached jack for %s\n", sdev->uid); + aaudio_cmd_property_listener(a, sdev->dev_id, sdev->dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_JACK_PLUGGED, 0)); + aaudio_handle_jack_connection_change(sdev); + } + + aaudio_reply_free(&buf); + + list_add_tail(&sdev->list, &a->subdevice_list); + return; + +fail: + aaudio_reply_free(&buf); + kfree(sdev); +} + +static void aaudio_init_stream_info(struct aaudio_subdevice *sdev, struct aaudio_stream *strm) +{ + if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_PHYS_FORMAT, 0), NULL, 0, + &strm->desc, sizeof(strm->desc))) + dev_warn(sdev->a->dev, "Failed to query stream descriptor\n"); + if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_GLOBAL, AAUDIO_PROP_LATENCY, 0), NULL, 0, &strm->latency, sizeof(u32))) + dev_warn(sdev->a->dev, "Failed to query stream latency\n"); + if (strm->desc.format_id == AAUDIO_FORMAT_LPCM) + sdev->is_pcm = true; +} + +static void aaudio_free_dev(struct aaudio_subdevice *sdev) +{ + size_t i; + for (i = 0; i < sdev->in_stream_cnt; i++) { + if (sdev->in_streams[i].alsa_hw_desc) + kfree(sdev->in_streams[i].alsa_hw_desc); + if (sdev->in_streams[i].buffers) + kfree(sdev->in_streams[i].buffers); + } + for (i = 0; i < sdev->out_stream_cnt; i++) { + if (sdev->out_streams[i].alsa_hw_desc) + kfree(sdev->out_streams[i].alsa_hw_desc); + if (sdev->out_streams[i].buffers) + kfree(sdev->out_streams[i].buffers); + } + kfree(sdev); +} + +static struct aaudio_subdevice *aaudio_find_dev_by_dev_id(struct aaudio_device *a, aaudio_device_id_t dev_id) +{ + struct aaudio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (dev_id == sdev->dev_id) + return sdev; + } + return NULL; +} + +static struct aaudio_subdevice *aaudio_find_dev_by_uid(struct aaudio_device *a, const char *uid) +{ + struct aaudio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (!strcmp(uid, sdev->uid)) + return sdev; + } + return NULL; +} + +static void aaudio_init_bs_stream(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm); +static void aaudio_init_bs_stream_host(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm); + +static int aaudio_init_bs(struct aaudio_device *a) +{ + int i, j; + struct aaudio_buffer_struct_device *dev; + struct aaudio_subdevice *sdev; + u32 ver, sig, bs_base; + + ver = ioread32(&a->reg_mem_gpr[0]); + if (ver < 3) { + dev_err(a->dev, "aaudio: Bad GPR version (%u)", ver); + return -EINVAL; + } + sig = ioread32(&a->reg_mem_gpr[1]); + if (sig != AAUDIO_SIG) { + dev_err(a->dev, "aaudio: Bad GPR sig (%x)", sig); + return -EINVAL; + } + bs_base = ioread32(&a->reg_mem_gpr[2]); + a->bs = (struct aaudio_buffer_struct *) ((u8 *) a->reg_mem_bs + bs_base); + if (a->bs->signature != AAUDIO_SIG) { + dev_err(a->dev, "aaudio: Bad BufferStruct sig (%x)", a->bs->signature); + return -EINVAL; + } + dev_info(a->dev, "aaudio: BufferStruct ver = %i\n", a->bs->version); + dev_info(a->dev, "aaudio: Num devices = %i\n", a->bs->num_devices); + for (i = 0; i < a->bs->num_devices; i++) { + dev = &a->bs->devices[i]; + dev_info(a->dev, "aaudio: Device %i %s\n", i, dev->name); + + sdev = aaudio_find_dev_by_uid(a, dev->name); + if (!sdev) { + dev_err(a->dev, "aaudio: Subdevice not found for BufferStruct device %s\n", dev->name); + continue; + } + sdev->buf_id = (u8) i; + dev->num_input_streams = 0; + for (j = 0; j < dev->num_output_streams; j++) { + dev_info(a->dev, "aaudio: Device %i Stream %i: Output; Buffer Count = %i\n", i, j, + dev->output_streams[j].num_buffers); + if (j < sdev->out_stream_cnt) + aaudio_init_bs_stream(a, &sdev->out_streams[j], &dev->output_streams[j]); + } + } + + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->buf_id != AAUDIO_BUFFER_ID_NONE) + continue; + sdev->buf_id = i; + dev_info(a->dev, "aaudio: Created device %i %s\n", i, sdev->uid); + strcpy(a->bs->devices[i].name, sdev->uid); + a->bs->devices[i].num_input_streams = 0; + a->bs->devices[i].num_output_streams = 0; + a->bs->num_devices = ++i; + } + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->in_stream_cnt == 1) { + dev_info(a->dev, "aaudio: Device %i Host Stream; Input\n", sdev->buf_id); + aaudio_init_bs_stream_host(a, &sdev->in_streams[0], &a->bs->devices[sdev->buf_id].input_streams[0]); + a->bs->devices[sdev->buf_id].num_input_streams = 1; + wmb(); + + if (aaudio_cmd_set_input_stream_address_ranges(a, sdev->dev_id)) { + dev_err(a->dev, "aaudio: Failed to set input stream address ranges\n"); + } + } + } + + return 0; +} + +static void aaudio_init_bs_stream(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm) +{ + size_t i; + strm->buffer_cnt = bs_strm->num_buffers; + if (bs_strm->num_buffers > AAUDIO_DEIVCE_MAX_BUFFER_COUNT) { + dev_warn(a->dev, "BufferStruct buffer count %u exceeds driver limit of %u\n", bs_strm->num_buffers, + AAUDIO_DEIVCE_MAX_BUFFER_COUNT); + strm->buffer_cnt = AAUDIO_DEIVCE_MAX_BUFFER_COUNT; + } + if (!strm->buffer_cnt) + return; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct aaudio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + for (i = 0; i < strm->buffer_cnt; i++) { + strm->buffers[i].dma_addr = a->reg_mem_bs_dma + (dma_addr_t) bs_strm->buffers[i].address; + strm->buffers[i].ptr = a->reg_mem_bs + bs_strm->buffers[i].address; + strm->buffers[i].size = bs_strm->buffers[i].size; + } + + if (strm->buffer_cnt == 1) { + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (aaudio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } + } +} + +static void aaudio_init_bs_stream_host(struct aaudio_device *a, struct aaudio_stream *strm, + struct aaudio_buffer_struct_stream *bs_strm) +{ + size_t size; + dma_addr_t dma_addr; + void *dma_ptr; + size = strm->desc.bytes_per_packet * 16640; + dma_ptr = dma_alloc_coherent(&a->pci->dev, size, &dma_addr, GFP_KERNEL); + if (!dma_ptr) { + dev_err(a->dev, "dma_alloc_coherent failed\n"); + return; + } + bs_strm->buffers[0].address = dma_addr; + bs_strm->buffers[0].size = size; + bs_strm->num_buffers = 1; + + memset(dma_ptr, 0, size); + + strm->buffer_cnt = 1; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct aaudio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + strm->buffers[0].dma_addr = dma_addr; + strm->buffers[0].ptr = dma_ptr; + strm->buffers[0].size = size; + + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (aaudio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } +} + +static void aaudio_handle_prop_change(struct aaudio_device *a, struct aaudio_msg *msg); + +void aaudio_handle_notification(struct aaudio_device *a, struct aaudio_msg *msg) +{ + struct aaudio_send_ctx sctx; + struct aaudio_msg_base base; + if (aaudio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case AAUDIO_MSG_NOTIFICATION_BOOT: + dev_info(a->dev, "Received boot notification from remote\n"); + + /* Resend the alive notify */ + if (aaudio_send(a, &sctx, 500, + aaudio_msg_write_alive_notification, 1, 3)) { + pr_err("Sending alive notification failed\n"); + } + break; + case AAUDIO_MSG_NOTIFICATION_ALIVE: + dev_info(a->dev, "Received alive notification from remote\n"); + complete_all(&a->remote_alive); + break; + case AAUDIO_MSG_PROPERTY_CHANGED: + aaudio_handle_prop_change(a, msg); + break; + default: + dev_info(a->dev, "Unhandled notification %i", base.msg); + break; + } +} + +struct aaudio_prop_change_work_struct { + struct work_struct ws; + struct aaudio_device *a; + aaudio_device_id_t dev; + aaudio_object_id_t obj; + struct aaudio_prop_addr prop; +}; + +static void aaudio_handle_jack_connection_change(struct aaudio_subdevice *sdev) +{ + u32 plugged; + if (!sdev->jack) + return; + /* NOTE: Apple made the plug status scoped to the input and output streams. This makes no sense for us, so I just + * always pick the OUTPUT status. */ + if (aaudio_cmd_get_primitive_property(sdev->a, sdev->dev_id, sdev->dev_id, + AAUDIO_PROP(AAUDIO_PROP_SCOPE_OUTPUT, AAUDIO_PROP_JACK_PLUGGED, 0), NULL, 0, &plugged, sizeof(plugged))) { + dev_err(sdev->a->dev, "Failed to get jack enable status\n"); + return; + } + dev_dbg(sdev->a->dev, "Jack is now %s\n", plugged ? "plugged" : "unplugged"); + snd_jack_report(sdev->jack, plugged ? sdev->jack->type : 0); +} + +void aaudio_handle_prop_change_work(struct work_struct *ws) +{ + struct aaudio_prop_change_work_struct *work = container_of(ws, struct aaudio_prop_change_work_struct, ws); + struct aaudio_subdevice *sdev; + + sdev = aaudio_find_dev_by_dev_id(work->a, work->dev); + if (!sdev) { + dev_err(work->a->dev, "Property notification change: device not found\n"); + goto done; + } + dev_dbg(work->a->dev, "Property changed for device: %s\n", sdev->uid); + + if (work->prop.scope == AAUDIO_PROP_SCOPE_OUTPUT && work->prop.selector == AAUDIO_PROP_JACK_PLUGGED) { + aaudio_handle_jack_connection_change(sdev); + } + +done: + kfree(work); +} + +void aaudio_handle_prop_change(struct aaudio_device *a, struct aaudio_msg *msg) +{ + /* NOTE: This is a scheduled work because this callback will generally need to query device information and this + * is not possible when we are in the reply parsing code's context. */ + struct aaudio_prop_change_work_struct *work; + work = kmalloc(sizeof(struct aaudio_prop_change_work_struct), GFP_KERNEL); + work->a = a; + INIT_WORK(&work->ws, aaudio_handle_prop_change_work); + aaudio_msg_read_property_changed(msg, &work->dev, &work->obj, &work->prop); + schedule_work(&work->ws); +} + +#define aaudio_send_cmd_response(a, sctx, msg, fn, ...) \ + if (aaudio_send_with_tag(a, sctx, ((struct aaudio_msg_header *) msg->data)->tag, 500, fn, ##__VA_ARGS__)) \ + pr_err("aaudio: Failed to reply to a command\n"); + +void aaudio_handle_cmd_timestamp(struct aaudio_device *a, struct aaudio_msg *msg) +{ + ktime_t time_os = ktime_get_boottime(); + struct aaudio_send_ctx sctx; + struct aaudio_subdevice *sdev; + u64 devid, timestamp, update_seed; + aaudio_msg_read_update_timestamp(msg, &devid, ×tamp, &update_seed); + dev_dbg(a->dev, "Received timestamp update for dev=%llx ts=%llx seed=%llx\n", devid, timestamp, update_seed); + + sdev = aaudio_find_dev_by_dev_id(a, devid); + aaudio_handle_timestamp(sdev, time_os, timestamp); + + aaudio_send_cmd_response(a, &sctx, msg, + aaudio_msg_write_update_timestamp_response); +} + +void aaudio_handle_command(struct aaudio_device *a, struct aaudio_msg *msg) +{ + struct aaudio_msg_base base; + if (aaudio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case AAUDIO_MSG_UPDATE_TIMESTAMP: + aaudio_handle_cmd_timestamp(a, msg); + break; + default: + dev_info(a->dev, "Unhandled device command %i", base.msg); + break; + } +} + +static struct pci_device_id aaudio_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1803) }, + { 0, }, +}; + +struct dev_pm_ops aaudio_pci_driver_pm = { + .suspend = aaudio_suspend, + .resume = aaudio_resume +}; +struct pci_driver aaudio_pci_driver = { + .name = "aaudio", + .id_table = aaudio_ids, + .probe = aaudio_probe, + .remove = aaudio_remove, + .driver = { + .pm = &aaudio_pci_driver_pm + } +}; + + +int aaudio_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&aaudio_chrdev, 0, 1, "aaudio"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + aaudio_class = class_create(THIS_MODULE, "aaudio"); +#else + aaudio_class = class_create("aaudio"); +#endif + if (IS_ERR(aaudio_class)) { + result = PTR_ERR(aaudio_class); + goto fail_class; + } + + result = pci_register_driver(&aaudio_pci_driver); + if (result) + goto fail_drv; + return 0; + +fail_drv: + pci_unregister_driver(&aaudio_pci_driver); +fail_class: + class_destroy(aaudio_class); +fail_chrdev: + unregister_chrdev_region(aaudio_chrdev, 1); + if (!result) + result = -EINVAL; + return result; +} + +void aaudio_module_exit(void) +{ + pci_unregister_driver(&aaudio_pci_driver); + class_destroy(aaudio_class); + unregister_chrdev_region(aaudio_chrdev, 1); +} + +struct aaudio_alsa_pcm_id_mapping aaudio_alsa_id_mappings[] = { + {"Speaker", 0}, + {"Digital Mic", 1}, + {"Codec Output", 2}, + {"Codec Input", 3}, + {"Bridge Loopback", 4}, + {} +}; + +module_param_named(index, aaudio_alsa_index, int, 0444); +MODULE_PARM_DESC(index, "Index value for Apple Internal Audio soundcard."); +module_param_named(id, aaudio_alsa_id, charp, 0444); +MODULE_PARM_DESC(id, "ID string for Apple Internal Audio soundcard."); diff --git a/drivers/staging/apple-bce/audio/audio.h b/drivers/staging/apple-bce/audio/audio.h new file mode 100644 index 000000000000..004bc1e22ea4 --- /dev/null +++ b/drivers/staging/apple-bce/audio/audio.h @@ -0,0 +1,125 @@ +#ifndef AAUDIO_H +#define AAUDIO_H + +#include +#include +#include "../apple_bce.h" +#include "protocol_bce.h" +#include "description.h" + +#define AAUDIO_SIG 0x19870423 + +#define AAUDIO_DEVICE_MAX_UID_LEN 128 +#define AAUDIO_DEIVCE_MAX_INPUT_STREAMS 1 +#define AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS 1 +#define AAUDIO_DEIVCE_MAX_BUFFER_COUNT 1 + +#define AAUDIO_BUFFER_ID_NONE 0xffu + +struct snd_card; +struct snd_pcm; +struct snd_pcm_hardware; +struct snd_jack; + +struct __attribute__((packed)) __attribute__((aligned(4))) aaudio_buffer_struct_buffer { + size_t address; + size_t size; + size_t pad[4]; +}; +struct aaudio_buffer_struct_stream { + u8 num_buffers; + struct aaudio_buffer_struct_buffer buffers[100]; + char filler[32]; +}; +struct aaudio_buffer_struct_device { + char name[128]; + u8 num_input_streams; + u8 num_output_streams; + struct aaudio_buffer_struct_stream input_streams[5]; + struct aaudio_buffer_struct_stream output_streams[5]; + char filler[128]; +}; +struct aaudio_buffer_struct { + u32 version; + u32 signature; + u32 flags; + u8 num_devices; + struct aaudio_buffer_struct_device devices[20]; +}; + +struct aaudio_device; +struct aaudio_dma_buf { + dma_addr_t dma_addr; + void *ptr; + size_t size; +}; +struct aaudio_stream { + aaudio_object_id_t id; + size_t buffer_cnt; + struct aaudio_dma_buf *buffers; + + struct aaudio_apple_description desc; + struct snd_pcm_hardware *alsa_hw_desc; + u32 latency; + + bool waiting_for_first_ts; + + ktime_t remote_timestamp; + snd_pcm_sframes_t frame_min; + int started; +}; +struct aaudio_subdevice { + struct aaudio_device *a; + struct list_head list; + aaudio_device_id_t dev_id; + u32 in_latency, out_latency; + u8 buf_id; + int alsa_id; + char uid[AAUDIO_DEVICE_MAX_UID_LEN + 1]; + size_t in_stream_cnt; + struct aaudio_stream in_streams[AAUDIO_DEIVCE_MAX_INPUT_STREAMS]; + size_t out_stream_cnt; + struct aaudio_stream out_streams[AAUDIO_DEIVCE_MAX_OUTPUT_STREAMS]; + bool is_pcm; + struct snd_pcm *pcm; + struct snd_jack *jack; +}; +struct aaudio_alsa_pcm_id_mapping { + const char *name; + int alsa_id; +}; + +struct aaudio_device { + struct pci_dev *pci; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_bs; + dma_addr_t reg_mem_bs_dma; + void __iomem *reg_mem_cfg; + + u32 __iomem *reg_mem_gpr; + + struct aaudio_buffer_struct *bs; + + struct apple_bce_device *bce; + struct aaudio_bce bcem; + + struct snd_card *card; + + struct list_head subdevice_list; + int next_alsa_id; + + struct completion remote_alive; +}; + +void aaudio_handle_notification(struct aaudio_device *a, struct aaudio_msg *msg); +void aaudio_handle_prop_change_work(struct work_struct *ws); +void aaudio_handle_cmd_timestamp(struct aaudio_device *a, struct aaudio_msg *msg); +void aaudio_handle_command(struct aaudio_device *a, struct aaudio_msg *msg); + +int aaudio_module_init(void); +void aaudio_module_exit(void); + +extern struct aaudio_alsa_pcm_id_mapping aaudio_alsa_id_mappings[]; + +#endif //AAUDIO_H diff --git a/drivers/staging/apple-bce/audio/description.h b/drivers/staging/apple-bce/audio/description.h new file mode 100644 index 000000000000..dfef3ab68f27 --- /dev/null +++ b/drivers/staging/apple-bce/audio/description.h @@ -0,0 +1,42 @@ +#ifndef AAUDIO_DESCRIPTION_H +#define AAUDIO_DESCRIPTION_H + +#include + +struct aaudio_apple_description { + u64 sample_rate_double; + u32 format_id; + u32 format_flags; + u32 bytes_per_packet; + u32 frames_per_packet; + u32 bytes_per_frame; + u32 channels_per_frame; + u32 bits_per_channel; + u32 reserved; +}; + +enum { + AAUDIO_FORMAT_LPCM = 0x6c70636d // 'lpcm' +}; + +enum { + AAUDIO_FORMAT_FLAG_FLOAT = 1, + AAUDIO_FORMAT_FLAG_BIG_ENDIAN = 2, + AAUDIO_FORMAT_FLAG_SIGNED = 4, + AAUDIO_FORMAT_FLAG_PACKED = 8, + AAUDIO_FORMAT_FLAG_ALIGNED_HIGH = 16, + AAUDIO_FORMAT_FLAG_NON_INTERLEAVED = 32, + AAUDIO_FORMAT_FLAG_NON_MIXABLE = 64 +}; + +static inline u64 aaudio_double_to_u64(u64 d) +{ + u8 sign = (u8) ((d >> 63) & 1); + s32 exp = (s32) ((d >> 52) & 0x7ff) - 1023; + u64 fr = d & ((1LL << 52) - 1); + if (sign || exp < 0) + return 0; + return (u64) ((1LL << exp) + (fr >> (52 - exp))); +} + +#endif //AAUDIO_DESCRIPTION_H diff --git a/drivers/staging/apple-bce/audio/pcm.c b/drivers/staging/apple-bce/audio/pcm.c new file mode 100644 index 000000000000..1026e10a9ac5 --- /dev/null +++ b/drivers/staging/apple-bce/audio/pcm.c @@ -0,0 +1,308 @@ +#include "pcm.h" +#include "audio.h" + +static u64 aaudio_get_alsa_fmtbit(struct aaudio_apple_description *desc) +{ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_FLOAT) { + if (desc->bits_per_channel == 32) { + if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT_LE; + } else if (desc->bits_per_channel == 64) { + if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT64_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT64_LE; + } else { + pr_err("aaudio: unsupported bits per channel for float format: %u\n", desc->bits_per_channel); + return 0; + } + } +#define DEFINE_BPC_OPTION(val, b) \ + case val: \ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_BIG_ENDIAN) { \ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## BE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## BE; \ + } else { \ + if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## LE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## LE; \ + } + if (desc->format_flags & AAUDIO_FORMAT_FLAG_PACKED) { + switch (desc->bits_per_channel) { + case 8: + case 16: + case 32: + break; + DEFINE_BPC_OPTION(24, 24_3) + default: + pr_err("aaudio: unsupported bits per channel for packed format: %u\n", desc->bits_per_channel); + return 0; + } + } + if (desc->format_flags & AAUDIO_FORMAT_FLAG_ALIGNED_HIGH) { + switch (desc->bits_per_channel) { + DEFINE_BPC_OPTION(24, 32_) + default: + pr_err("aaudio: unsupported bits per channel for high-aligned format: %u\n", desc->bits_per_channel); + return 0; + } + } + switch (desc->bits_per_channel) { + case 8: + if (desc->format_flags & AAUDIO_FORMAT_FLAG_SIGNED) + return SNDRV_PCM_FMTBIT_S8; + else + return SNDRV_PCM_FMTBIT_U8; + DEFINE_BPC_OPTION(16, 16_) + DEFINE_BPC_OPTION(24, 24_) + DEFINE_BPC_OPTION(32, 32_) + default: + pr_err("aaudio: unsupported bits per channel: %u\n", desc->bits_per_channel); + return 0; + } +} +int aaudio_create_hw_info(struct aaudio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, + size_t buf_size) +{ + uint rate; + alsa_hw->info = (SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_BLOCK_TRANSFER | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_DOUBLE); + if (desc->format_flags & AAUDIO_FORMAT_FLAG_NON_MIXABLE) + pr_warn("aaudio: unsupported hw flag: NON_MIXABLE\n"); + if (!(desc->format_flags & AAUDIO_FORMAT_FLAG_NON_INTERLEAVED)) + alsa_hw->info |= SNDRV_PCM_INFO_INTERLEAVED; + alsa_hw->formats = aaudio_get_alsa_fmtbit(desc); + if (!alsa_hw->formats) + return -EINVAL; + rate = (uint) aaudio_double_to_u64(desc->sample_rate_double); + alsa_hw->rates = snd_pcm_rate_to_rate_bit(rate); + alsa_hw->rate_min = rate; + alsa_hw->rate_max = rate; + alsa_hw->channels_min = desc->channels_per_frame; + alsa_hw->channels_max = desc->channels_per_frame; + alsa_hw->buffer_bytes_max = buf_size; + alsa_hw->period_bytes_min = desc->bytes_per_packet; + alsa_hw->period_bytes_max = desc->bytes_per_packet; + alsa_hw->periods_min = (uint) (buf_size / desc->bytes_per_packet); + alsa_hw->periods_max = (uint) (buf_size / desc->bytes_per_packet); + pr_debug("aaudio_create_hw_info: format = %llu, rate = %u/%u. channels = %u, periods = %u, period size = %lu\n", + alsa_hw->formats, alsa_hw->rate_min, alsa_hw->rates, alsa_hw->channels_min, alsa_hw->periods_min, + alsa_hw->period_bytes_min); + return 0; +} + +static struct aaudio_stream *aaudio_pcm_stream(struct snd_pcm_substream *substream) +{ + struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + return &sdev->out_streams[substream->number]; + else + return &sdev->in_streams[substream->number]; +} + +static int aaudio_pcm_open(struct snd_pcm_substream *substream) +{ + pr_debug("aaudio_pcm_open\n"); + substream->runtime->hw = *aaudio_pcm_stream(substream)->alsa_hw_desc; + + return 0; +} + +static int aaudio_pcm_close(struct snd_pcm_substream *substream) +{ + pr_debug("aaudio_pcm_close\n"); + return 0; +} + +static int aaudio_pcm_prepare(struct snd_pcm_substream *substream) +{ + return 0; +} + +static int aaudio_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) +{ + struct aaudio_stream *astream = aaudio_pcm_stream(substream); + pr_debug("aaudio_pcm_hw_params\n"); + + if (!astream->buffer_cnt || !astream->buffers) + return -EINVAL; + + substream->runtime->dma_area = astream->buffers[0].ptr; + substream->runtime->dma_addr = astream->buffers[0].dma_addr; + substream->runtime->dma_bytes = astream->buffers[0].size; + return 0; +} + +static int aaudio_pcm_hw_free(struct snd_pcm_substream *substream) +{ + pr_debug("aaudio_pcm_hw_free\n"); + return 0; +} + +static void aaudio_pcm_start(struct snd_pcm_substream *substream) +{ + struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct aaudio_stream *stream = aaudio_pcm_stream(substream); + void *buf; + size_t s; + ktime_t time_start, time_end; + bool back_buffer; + time_start = ktime_get(); + + back_buffer = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); + + if (back_buffer) { + s = frames_to_bytes(substream->runtime, substream->runtime->control->appl_ptr); + buf = kmalloc(s, GFP_KERNEL); + memcpy_fromio(buf, substream->runtime->dma_area, s); + time_end = ktime_get(); + pr_debug("aaudio: Backed up the buffer in %lluns [%li]\n", ktime_to_ns(time_end - time_start), + substream->runtime->control->appl_ptr); + } + + stream->waiting_for_first_ts = true; + stream->frame_min = stream->latency; + + aaudio_cmd_start_io(sdev->a, sdev->dev_id); + if (back_buffer) + memcpy_toio(substream->runtime->dma_area, buf, s); + + time_end = ktime_get(); + pr_debug("aaudio: Started the audio device in %lluns\n", ktime_to_ns(time_end - time_start)); +} + +static int aaudio_pcm_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct aaudio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct aaudio_stream *stream = aaudio_pcm_stream(substream); + pr_debug("aaudio_pcm_trigger %x\n", cmd); + + /* We only supports triggers on the #0 buffer */ + if (substream->number != 0) + return 0; + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + aaudio_pcm_start(substream); + stream->started = 1; + break; + case SNDRV_PCM_TRIGGER_STOP: + aaudio_cmd_stop_io(sdev->a, sdev->dev_id); + stream->started = 0; + break; + default: + return -EINVAL; + } + return 0; +} + +static snd_pcm_uframes_t aaudio_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct aaudio_stream *stream = aaudio_pcm_stream(substream); + ktime_t time_from_start; + snd_pcm_sframes_t frames; + snd_pcm_sframes_t buffer_time_length; + + if (!stream->started || stream->waiting_for_first_ts) { + pr_warn("aaudio_pcm_pointer while not started\n"); + return 0; + } + + /* Approximate the pointer based on the last received timestamp */ + time_from_start = ktime_get_boottime() - stream->remote_timestamp; + buffer_time_length = NSEC_PER_SEC * substream->runtime->buffer_size / substream->runtime->rate; + frames = (ktime_to_ns(time_from_start) % buffer_time_length) * substream->runtime->buffer_size / buffer_time_length; + if (ktime_to_ns(time_from_start) < buffer_time_length) { + if (frames < stream->frame_min) + frames = stream->frame_min; + else + stream->frame_min = 0; + } else { + if (ktime_to_ns(time_from_start) < 2 * buffer_time_length) + stream->frame_min = frames; + else + stream->frame_min = 0; /* Heavy desync */ + } + frames -= stream->latency; + if (frames < 0) + frames += ((-frames - 1) / substream->runtime->buffer_size + 1) * substream->runtime->buffer_size; + return (snd_pcm_uframes_t) frames; +} + +static struct snd_pcm_ops aaudio_pcm_ops = { + .open = aaudio_pcm_open, + .close = aaudio_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = aaudio_pcm_hw_params, + .hw_free = aaudio_pcm_hw_free, + .prepare = aaudio_pcm_prepare, + .trigger = aaudio_pcm_trigger, + .pointer = aaudio_pcm_pointer, + .mmap = snd_pcm_lib_mmap_iomem +}; + +int aaudio_create_pcm(struct aaudio_subdevice *sdev) +{ + struct snd_pcm *pcm; + struct aaudio_alsa_pcm_id_mapping *id_mapping; + int err; + + if (!sdev->is_pcm || (sdev->in_stream_cnt == 0 && sdev->out_stream_cnt == 0)) { + return -EINVAL; + } + + for (id_mapping = aaudio_alsa_id_mappings; id_mapping->name; id_mapping++) { + if (!strcmp(sdev->uid, id_mapping->name)) { + sdev->alsa_id = id_mapping->alsa_id; + break; + } + } + if (!id_mapping->name) + sdev->alsa_id = sdev->a->next_alsa_id++; + err = snd_pcm_new(sdev->a->card, sdev->uid, sdev->alsa_id, + (int) sdev->out_stream_cnt, (int) sdev->in_stream_cnt, &pcm); + if (err < 0) + return err; + pcm->private_data = sdev; + pcm->nonatomic = 1; + sdev->pcm = pcm; + strcpy(pcm->name, sdev->uid); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &aaudio_pcm_ops); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &aaudio_pcm_ops); + return 0; +} + +static void aaudio_handle_stream_timestamp(struct snd_pcm_substream *substream, ktime_t timestamp) +{ + unsigned long flags; + struct aaudio_stream *stream; + + stream = aaudio_pcm_stream(substream); + snd_pcm_stream_lock_irqsave(substream, flags); + stream->remote_timestamp = timestamp; + if (stream->waiting_for_first_ts) { + stream->waiting_for_first_ts = false; + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + snd_pcm_stream_unlock_irqrestore(substream, flags); + snd_pcm_period_elapsed(substream); +} + +void aaudio_handle_timestamp(struct aaudio_subdevice *sdev, ktime_t os_timestamp, u64 dev_timestamp) +{ + struct snd_pcm_substream *substream; + + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (substream) + aaudio_handle_stream_timestamp(substream, dev_timestamp); + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + if (substream) + aaudio_handle_stream_timestamp(substream, os_timestamp); +} diff --git a/drivers/staging/apple-bce/audio/pcm.h b/drivers/staging/apple-bce/audio/pcm.h new file mode 100644 index 000000000000..ea5f35fbe408 --- /dev/null +++ b/drivers/staging/apple-bce/audio/pcm.h @@ -0,0 +1,16 @@ +#ifndef AAUDIO_PCM_H +#define AAUDIO_PCM_H + +#include +#include + +struct aaudio_subdevice; +struct aaudio_apple_description; +struct snd_pcm_hardware; + +int aaudio_create_hw_info(struct aaudio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, size_t buf_size); +int aaudio_create_pcm(struct aaudio_subdevice *sdev); + +void aaudio_handle_timestamp(struct aaudio_subdevice *sdev, ktime_t os_timestamp, u64 dev_timestamp); + +#endif //AAUDIO_PCM_H diff --git a/drivers/staging/apple-bce/audio/protocol.c b/drivers/staging/apple-bce/audio/protocol.c new file mode 100644 index 000000000000..2314813aeead --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol.c @@ -0,0 +1,347 @@ +#include "protocol.h" +#include "protocol_bce.h" +#include "audio.h" + +int aaudio_msg_read_base(struct aaudio_msg *msg, struct aaudio_msg_base *base) +{ + if (msg->size < sizeof(struct aaudio_msg_header) + sizeof(struct aaudio_msg_base) * 2) + return -EINVAL; + *base = *((struct aaudio_msg_base *) ((struct aaudio_msg_header *) msg->data + 1)); + return 0; +} + +#define READ_START(type) \ + size_t offset = sizeof(struct aaudio_msg_header) + sizeof(struct aaudio_msg_base); (void)offset; \ + if (((struct aaudio_msg_base *) ((struct aaudio_msg_header *) msg->data + 1))->msg != type) \ + return -EINVAL; +#define READ_DEVID_VAR(devid) *devid = ((struct aaudio_msg_header *) msg->data)->device_id +#define READ_VAL(type) ({ offset += sizeof(type); *((type *) ((u8 *) msg->data + offset - sizeof(type))); }) +#define READ_VAR(type, var) *var = READ_VAL(type) + +int aaudio_msg_read_start_io_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_START_IO_RESPONSE); + return 0; +} + +int aaudio_msg_read_stop_io_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_STOP_IO_RESPONSE); + return 0; +} + +int aaudio_msg_read_update_timestamp(struct aaudio_msg *msg, aaudio_device_id_t *devid, + u64 *timestamp, u64 *update_seed) +{ + READ_START(AAUDIO_MSG_UPDATE_TIMESTAMP); + READ_DEVID_VAR(devid); + READ_VAR(u64, timestamp); + READ_VAR(u64, update_seed); + return 0; +} + +int aaudio_msg_read_get_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop, void **data, u64 *data_size) +{ + READ_START(AAUDIO_MSG_GET_PROPERTY_RESPONSE); + READ_VAR(aaudio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + READ_VAR(u64, data_size); + *data = ((u8 *) msg->data + offset); + /* offset += data_size; */ + return 0; +} + +int aaudio_msg_read_set_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj) +{ + READ_START(AAUDIO_MSG_SET_PROPERTY_RESPONSE); + READ_VAR(aaudio_object_id_t, obj); + return 0; +} + +int aaudio_msg_read_property_listener_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop) +{ + READ_START(AAUDIO_MSG_PROPERTY_LISTENER_RESPONSE); + READ_VAR(aaudio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int aaudio_msg_read_property_changed(struct aaudio_msg *msg, aaudio_device_id_t *devid, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop) +{ + READ_START(AAUDIO_MSG_PROPERTY_CHANGED); + READ_DEVID_VAR(devid); + READ_VAR(aaudio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int aaudio_msg_read_set_input_stream_address_ranges_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE); + return 0; +} + +int aaudio_msg_read_get_input_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(AAUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(aaudio_object_id_t); */ + return 0; +} + +int aaudio_msg_read_get_output_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(AAUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(aaudio_object_id_t); */ + return 0; +} + +int aaudio_msg_read_set_remote_access_response(struct aaudio_msg *msg) +{ + READ_START(AAUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE); + return 0; +} + +int aaudio_msg_read_get_device_list_response(struct aaudio_msg *msg, aaudio_device_id_t **dev_l, u64 *dev_cnt) +{ + READ_START(AAUDIO_MSG_GET_DEVICE_LIST_RESPONSE); + READ_VAR(u64, dev_cnt); + *dev_l = (aaudio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += dev_cnt * sizeof(aaudio_device_id_t); */ + return 0; +} + +#define WRITE_START_OF_TYPE(typev, devid) \ + size_t offset = sizeof(struct aaudio_msg_header); (void) offset; \ + ((struct aaudio_msg_header *) msg->data)->type = (typev); \ + ((struct aaudio_msg_header *) msg->data)->device_id = (devid); +#define WRITE_START_COMMAND(devid) WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_COMMAND, devid) +#define WRITE_START_RESPONSE() WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_RESPONSE, 0) +#define WRITE_START_NOTIFICATION() WRITE_START_OF_TYPE(AAUDIO_MSG_TYPE_NOTIFICATION, 0) +#define WRITE_VAL(type, value) { *((type *) ((u8 *) msg->data + offset)) = value; offset += sizeof(value); } +#define WRITE_BIN(value, size) { memcpy((u8 *) msg->data + offset, value, size); offset += size; } +#define WRITE_BASE(type) WRITE_VAL(u32, type) WRITE_VAL(u32, 0) +#define WRITE_END() { msg->size = offset; } + +void aaudio_msg_write_start_io(struct aaudio_msg *msg, aaudio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_START_IO); + WRITE_END(); +} + +void aaudio_msg_write_stop_io(struct aaudio_msg *msg, aaudio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_STOP_IO); + WRITE_END(); +} + +void aaudio_msg_write_get_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_GET_PROPERTY); + WRITE_VAL(aaudio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void aaudio_msg_write_set_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_SET_PROPERTY); + WRITE_VAL(aaudio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, data_size); + WRITE_BIN(data, data_size); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void aaudio_msg_write_property_listener(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(AAUDIO_MSG_PROPERTY_LISTENER); + WRITE_VAL(aaudio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_END(); +} + +void aaudio_msg_write_set_input_stream_address_ranges(struct aaudio_msg *msg, aaudio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES); + WRITE_END(); +} + +void aaudio_msg_write_get_input_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(AAUDIO_MSG_GET_INPUT_STREAM_LIST); + WRITE_END(); +} + +void aaudio_msg_write_get_output_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(AAUDIO_MSG_GET_OUTPUT_STREAM_LIST); + WRITE_END(); +} + +void aaudio_msg_write_set_remote_access(struct aaudio_msg *msg, u64 mode) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(AAUDIO_MSG_SET_REMOTE_ACCESS); + WRITE_VAL(u64, mode); + WRITE_END(); +} + +void aaudio_msg_write_alive_notification(struct aaudio_msg *msg, u32 proto_ver, u32 msg_ver) +{ + WRITE_START_NOTIFICATION(); + WRITE_BASE(AAUDIO_MSG_NOTIFICATION_ALIVE); + WRITE_VAL(u32, proto_ver); + WRITE_VAL(u32, msg_ver); + WRITE_END(); +} + +void aaudio_msg_write_update_timestamp_response(struct aaudio_msg *msg) +{ + WRITE_START_RESPONSE(); + WRITE_BASE(AAUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE); + WRITE_END(); +} + +void aaudio_msg_write_get_device_list(struct aaudio_msg *msg) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(AAUDIO_MSG_GET_DEVICE_LIST); + WRITE_END(); +} + +#define CMD_SHARED_VARS_NO_REPLY \ + int status = 0; \ + struct aaudio_send_ctx sctx; +#define CMD_SHARED_VARS \ + CMD_SHARED_VARS_NO_REPLY \ + struct aaudio_msg reply = aaudio_reply_alloc(); \ + struct aaudio_msg *buf = &reply; +#define CMD_SEND_REQUEST(fn, ...) \ + if ((status = aaudio_send_cmd_sync(a, &sctx, buf, 500, fn, ##__VA_ARGS__))) \ + return status; +#define CMD_DEF_SHARED_AND_SEND(fn, ...) \ + CMD_SHARED_VARS \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_DEF_SHARED_NO_REPLY_AND_SEND(fn, ...) \ + CMD_SHARED_VARS_NO_REPLY \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_HNDL_REPLY_NO_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + return status; +#define CMD_HNDL_REPLY_AND_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + aaudio_reply_free(&reply); \ + return status; + +int aaudio_cmd_start_io(struct aaudio_device *a, aaudio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_start_io, devid); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_start_io_response); +} +int aaudio_cmd_stop_io(struct aaudio_device *a, aaudio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_stop_io, devid); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_stop_io_response); +} +int aaudio_cmd_get_property(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_property, devid, obj, prop, qualifier, qualifier_size); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_property_response, &obj, &prop, data, data_size); +} +int aaudio_cmd_get_primitive_property(struct aaudio_device *a, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + int status; + struct aaudio_msg reply = aaudio_reply_alloc(); + void *r_data; + u64 r_data_size; + if ((status = aaudio_cmd_get_property(a, &reply, devid, obj, prop, qualifier, qualifier_size, + &r_data, &r_data_size))) + goto finish; + if (r_data_size != data_size) { + status = -EINVAL; + goto finish; + } + memcpy(data, r_data, data_size); +finish: + aaudio_reply_free(&reply); + return status; +} +int aaudio_cmd_set_property(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_property, devid, obj, prop, data, data_size, + qualifier, qualifier_size); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_property_response, &obj); +} +int aaudio_cmd_property_listener(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_property_listener, devid, obj, prop); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_property_listener_response, &obj, &prop); +} +int aaudio_cmd_set_input_stream_address_ranges(struct aaudio_device *a, aaudio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_input_stream_address_ranges, devid); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_input_stream_address_ranges_response); +} +int aaudio_cmd_get_input_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_input_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_input_stream_list_response, str_l, str_cnt); +} +int aaudio_cmd_get_output_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_output_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_output_stream_list_response, str_l, str_cnt); +} +int aaudio_cmd_set_remote_access(struct aaudio_device *a, u64 mode) +{ + CMD_DEF_SHARED_AND_SEND(aaudio_msg_write_set_remote_access, mode); + CMD_HNDL_REPLY_AND_FREE(aaudio_msg_read_set_remote_access_response); +} +int aaudio_cmd_get_device_list(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t **dev_l, u64 *dev_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(aaudio_msg_write_get_device_list); + CMD_HNDL_REPLY_NO_FREE(aaudio_msg_read_get_device_list_response, dev_l, dev_cnt); +} \ No newline at end of file diff --git a/drivers/staging/apple-bce/audio/protocol.h b/drivers/staging/apple-bce/audio/protocol.h new file mode 100644 index 000000000000..3427486f3f57 --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol.h @@ -0,0 +1,147 @@ +#ifndef AAUDIO_PROTOCOL_H +#define AAUDIO_PROTOCOL_H + +#include + +struct aaudio_device; + +typedef u64 aaudio_device_id_t; +typedef u64 aaudio_object_id_t; + +struct aaudio_msg { + void *data; + size_t size; +}; + +struct __attribute__((packed)) aaudio_msg_header { + char tag[4]; + u8 type; + aaudio_device_id_t device_id; // Idk, use zero for commands? +}; +struct __attribute__((packed)) aaudio_msg_base { + u32 msg; + u32 status; +}; + +struct aaudio_prop_addr { + u32 scope; + u32 selector; + u32 element; +}; +#define AAUDIO_PROP(scope, sel, el) (struct aaudio_prop_addr) { scope, sel, el } + +enum { + AAUDIO_MSG_TYPE_COMMAND = 1, + AAUDIO_MSG_TYPE_RESPONSE = 2, + AAUDIO_MSG_TYPE_NOTIFICATION = 3 +}; + +enum { + AAUDIO_MSG_START_IO = 0, + AAUDIO_MSG_START_IO_RESPONSE = 1, + AAUDIO_MSG_STOP_IO = 2, + AAUDIO_MSG_STOP_IO_RESPONSE = 3, + AAUDIO_MSG_UPDATE_TIMESTAMP = 4, + AAUDIO_MSG_GET_PROPERTY = 7, + AAUDIO_MSG_GET_PROPERTY_RESPONSE = 8, + AAUDIO_MSG_SET_PROPERTY = 9, + AAUDIO_MSG_SET_PROPERTY_RESPONSE = 10, + AAUDIO_MSG_PROPERTY_LISTENER = 11, + AAUDIO_MSG_PROPERTY_LISTENER_RESPONSE = 12, + AAUDIO_MSG_PROPERTY_CHANGED = 13, + AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES = 18, + AAUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE = 19, + AAUDIO_MSG_GET_INPUT_STREAM_LIST = 24, + AAUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE = 25, + AAUDIO_MSG_GET_OUTPUT_STREAM_LIST = 26, + AAUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE = 27, + AAUDIO_MSG_SET_REMOTE_ACCESS = 32, + AAUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE = 33, + AAUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE = 34, + + AAUDIO_MSG_NOTIFICATION_ALIVE = 100, + AAUDIO_MSG_GET_DEVICE_LIST = 101, + AAUDIO_MSG_GET_DEVICE_LIST_RESPONSE = 102, + AAUDIO_MSG_NOTIFICATION_BOOT = 104 +}; + +enum { + AAUDIO_REMOTE_ACCESS_OFF = 0, + AAUDIO_REMOTE_ACCESS_ON = 2 +}; + +enum { + AAUDIO_PROP_SCOPE_GLOBAL = 0x676c6f62, // 'glob' + AAUDIO_PROP_SCOPE_INPUT = 0x696e7074, // 'inpt' + AAUDIO_PROP_SCOPE_OUTPUT = 0x6f757470 // 'outp' +}; + +enum { + AAUDIO_PROP_UID = 0x75696420, // 'uid ' + AAUDIO_PROP_BOOL_VALUE = 0x6263766c, // 'bcvl' + AAUDIO_PROP_JACK_PLUGGED = 0x6a61636b, // 'jack' + AAUDIO_PROP_SEL_VOLUME = 0x64656176, // 'deav' + AAUDIO_PROP_LATENCY = 0x6c746e63, // 'ltnc' + AAUDIO_PROP_PHYS_FORMAT = 0x70667420 // 'pft ' +}; + +int aaudio_msg_read_base(struct aaudio_msg *msg, struct aaudio_msg_base *base); + +int aaudio_msg_read_start_io_response(struct aaudio_msg *msg); +int aaudio_msg_read_stop_io_response(struct aaudio_msg *msg); +int aaudio_msg_read_update_timestamp(struct aaudio_msg *msg, aaudio_device_id_t *devid, + u64 *timestamp, u64 *update_seed); +int aaudio_msg_read_get_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop, void **data, u64 *data_size); +int aaudio_msg_read_set_property_response(struct aaudio_msg *msg, aaudio_object_id_t *obj); +int aaudio_msg_read_property_listener_response(struct aaudio_msg *msg,aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop); +int aaudio_msg_read_property_changed(struct aaudio_msg *msg, aaudio_device_id_t *devid, aaudio_object_id_t *obj, + struct aaudio_prop_addr *prop); +int aaudio_msg_read_set_input_stream_address_ranges_response(struct aaudio_msg *msg); +int aaudio_msg_read_get_input_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_msg_read_get_output_stream_list_response(struct aaudio_msg *msg, aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_msg_read_set_remote_access_response(struct aaudio_msg *msg); +int aaudio_msg_read_get_device_list_response(struct aaudio_msg *msg, aaudio_device_id_t **dev_l, u64 *dev_cnt); + +void aaudio_msg_write_start_io(struct aaudio_msg *msg, aaudio_device_id_t dev); +void aaudio_msg_write_stop_io(struct aaudio_msg *msg, aaudio_device_id_t dev); +void aaudio_msg_write_get_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size); +void aaudio_msg_write_set_property(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size); +void aaudio_msg_write_property_listener(struct aaudio_msg *msg, aaudio_device_id_t dev, aaudio_object_id_t obj, + struct aaudio_prop_addr prop); +void aaudio_msg_write_set_input_stream_address_ranges(struct aaudio_msg *msg, aaudio_device_id_t devid); +void aaudio_msg_write_get_input_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid); +void aaudio_msg_write_get_output_stream_list(struct aaudio_msg *msg, aaudio_device_id_t devid); +void aaudio_msg_write_set_remote_access(struct aaudio_msg *msg, u64 mode); +void aaudio_msg_write_alive_notification(struct aaudio_msg *msg, u32 proto_ver, u32 msg_ver); +void aaudio_msg_write_update_timestamp_response(struct aaudio_msg *msg); +void aaudio_msg_write_get_device_list(struct aaudio_msg *msg); + + +int aaudio_cmd_start_io(struct aaudio_device *a, aaudio_device_id_t devid); +int aaudio_cmd_stop_io(struct aaudio_device *a, aaudio_device_id_t devid); +int aaudio_cmd_get_property(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size); +int aaudio_cmd_get_primitive_property(struct aaudio_device *a, + aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int aaudio_cmd_set_property(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int aaudio_cmd_property_listener(struct aaudio_device *a, aaudio_device_id_t devid, aaudio_object_id_t obj, + struct aaudio_prop_addr prop); +int aaudio_cmd_set_input_stream_address_ranges(struct aaudio_device *a, aaudio_device_id_t devid); +int aaudio_cmd_get_input_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_cmd_get_output_stream_list(struct aaudio_device *a, struct aaudio_msg *buf, aaudio_device_id_t devid, + aaudio_object_id_t **str_l, u64 *str_cnt); +int aaudio_cmd_set_remote_access(struct aaudio_device *a, u64 mode); +int aaudio_cmd_get_device_list(struct aaudio_device *a, struct aaudio_msg *buf, + aaudio_device_id_t **dev_l, u64 *dev_cnt); + + + +#endif //AAUDIO_PROTOCOL_H diff --git a/drivers/staging/apple-bce/audio/protocol_bce.c b/drivers/staging/apple-bce/audio/protocol_bce.c new file mode 100644 index 000000000000..28f2dfd44d67 --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol_bce.c @@ -0,0 +1,226 @@ +#include "protocol_bce.h" + +#include "audio.h" + +static void aaudio_bce_out_queue_completion(struct bce_queue_sq *sq); +static void aaudio_bce_in_queue_completion(struct bce_queue_sq *sq); +static int aaudio_bce_queue_init(struct aaudio_device *dev, struct aaudio_bce_queue *q, const char *name, int direction, + bce_sq_completion cfn); +void aaudio_bce_in_queue_submit_pending(struct aaudio_bce_queue *q, size_t count); + +int aaudio_bce_init(struct aaudio_device *dev) +{ + int status; + struct aaudio_bce *bce = &dev->bcem; + bce->cq = bce_create_cq(dev->bce, 0x80); + spin_lock_init(&bce->spinlock); + if (!bce->cq) + return -EINVAL; + if ((status = aaudio_bce_queue_init(dev, &bce->qout, "com.apple.BridgeAudio.IntelToARM", DMA_TO_DEVICE, + aaudio_bce_out_queue_completion))) { + return status; + } + if ((status = aaudio_bce_queue_init(dev, &bce->qin, "com.apple.BridgeAudio.ARMToIntel", DMA_FROM_DEVICE, + aaudio_bce_in_queue_completion))) { + return status; + } + aaudio_bce_in_queue_submit_pending(&bce->qin, bce->qin.el_count); + return 0; +} + +int aaudio_bce_queue_init(struct aaudio_device *dev, struct aaudio_bce_queue *q, const char *name, int direction, + bce_sq_completion cfn) +{ + q->cq = dev->bcem.cq; + q->el_size = AAUDIO_BCE_QUEUE_ELEMENT_SIZE; + q->el_count = AAUDIO_BCE_QUEUE_ELEMENT_COUNT; + /* NOTE: The Apple impl uses 0x80 as the queue size, however we use 21 (in fact 20) to simplify the impl */ + q->sq = bce_create_sq(dev->bce, q->cq, name, (u32) (q->el_count + 1), direction, cfn, dev); + if (!q->sq) + return -EINVAL; + + q->data = dma_alloc_coherent(&dev->bce->pci->dev, q->el_size * q->el_count, &q->dma_addr, GFP_KERNEL); + if (!q->data) { + bce_destroy_sq(dev->bce, q->sq); + return -EINVAL; + } + return 0; +} + +static void aaudio_send_create_tag(struct aaudio_bce *b, int *tagn, char tag[4]) +{ + char tag_zero[5]; + b->tag_num = (b->tag_num + 1) % AAUDIO_BCE_QUEUE_TAG_COUNT; + *tagn = b->tag_num; + snprintf(tag_zero, 5, "S%03d", b->tag_num); + *((u32 *) tag) = *((u32 *) tag_zero); +} + +int __aaudio_send_prepare(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, char *tag) +{ + int status; + size_t index; + void *dptr; + struct aaudio_msg_header *header; + if ((status = bce_reserve_submission(b->qout.sq, &ctx->timeout))) + return status; + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + index = b->qout.data_tail; + dptr = (u8 *) b->qout.data + index * b->qout.el_size; + ctx->msg.data = dptr; + header = dptr; + if (tag) + *((u32 *) header->tag) = *((u32 *) tag); + else + aaudio_send_create_tag(b, &ctx->tag_n, header->tag); + return 0; +} + +void __aaudio_send(struct aaudio_bce *b, struct aaudio_send_ctx *ctx) +{ + struct bce_qe_submission *s = bce_next_submission(b->qout.sq); +#ifdef DEBUG + pr_debug("aaudio: Sending command data\n"); + print_hex_dump(KERN_DEBUG, "aaudio:OUT ", DUMP_PREFIX_NONE, 32, 1, ctx->msg.data, ctx->msg.size, true); +#endif + bce_set_submission_single(s, b->qout.dma_addr + (dma_addr_t) (ctx->msg.data - b->qout.data), ctx->msg.size); + bce_submit_to_device(b->qout.sq); + b->qout.data_tail = (b->qout.data_tail + 1) % b->qout.el_count; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); +} + +int __aaudio_send_cmd_sync(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, struct aaudio_msg *reply) +{ + struct aaudio_bce_queue_entry ent; + DECLARE_COMPLETION_ONSTACK(cmpl); + ent.msg = reply; + ent.cmpl = &cmpl; + b->pending_entries[ctx->tag_n] = &ent; + __aaudio_send(b, ctx); /* unlocks the spinlock */ + ctx->timeout = wait_for_completion_timeout(&cmpl, ctx->timeout); + if (ctx->timeout == 0) { + /* Remove the pending queue entry; this will be normally handled by the completion route but + * during a timeout it won't */ + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + if (b->pending_entries[ctx->tag_n] == &ent) + b->pending_entries[ctx->tag_n] = NULL; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); + return -ETIMEDOUT; + } + return 0; +} + +static void aaudio_handle_reply(struct aaudio_bce *b, struct aaudio_msg *reply) +{ + const char *tag; + int tagn; + unsigned long irq_flags; + char tag_zero[5]; + struct aaudio_bce_queue_entry *entry; + + tag = ((struct aaudio_msg_header *) reply->data)->tag; + if (tag[0] != 'S') { + pr_err("aaudio_handle_reply: Unexpected tag: %.4s\n", tag); + return; + } + *((u32 *) tag_zero) = *((u32 *) tag); + tag_zero[4] = 0; + if (kstrtoint(&tag_zero[1], 10, &tagn)) { + pr_err("aaudio_handle_reply: Tag parse failed: %.4s\n", tag); + return; + } + + spin_lock_irqsave(&b->spinlock, irq_flags); + entry = b->pending_entries[tagn]; + if (entry) { + if (reply->size < entry->msg->size) + entry->msg->size = reply->size; + memcpy(entry->msg->data, reply->data, entry->msg->size); + complete(entry->cmpl); + + b->pending_entries[tagn] = NULL; + } else { + pr_err("aaudio_handle_reply: No queued item found for tag: %.4s\n", tag); + } + spin_unlock_irqrestore(&b->spinlock, irq_flags); +} + +static void aaudio_bce_out_queue_completion(struct bce_queue_sq *sq) +{ + while (bce_next_completion(sq)) { + //pr_info("aaudio: Send confirmed\n"); + bce_notify_submission_complete(sq); + } +} + +static void aaudio_bce_in_queue_handle_msg(struct aaudio_device *a, struct aaudio_msg *msg); + +static void aaudio_bce_in_queue_completion(struct bce_queue_sq *sq) +{ + struct aaudio_msg msg; + struct aaudio_device *dev = sq->userdata; + struct aaudio_bce_queue *q = &dev->bcem.qin; + struct bce_sq_completion_data *c; + size_t cnt = 0; + + mb(); + while ((c = bce_next_completion(sq))) { + msg.data = (u8 *) q->data + q->data_head * q->el_size; + msg.size = c->data_size; +#ifdef DEBUG + pr_debug("aaudio: Received command data %llx\n", c->data_size); + print_hex_dump(KERN_DEBUG, "aaudio:IN ", DUMP_PREFIX_NONE, 32, 1, msg.data, min(msg.size, 128UL), true); +#endif + aaudio_bce_in_queue_handle_msg(dev, &msg); + + q->data_head = (q->data_head + 1) % q->el_count; + + bce_notify_submission_complete(sq); + ++cnt; + } + aaudio_bce_in_queue_submit_pending(q, cnt); +} + +static void aaudio_bce_in_queue_handle_msg(struct aaudio_device *a, struct aaudio_msg *msg) +{ + struct aaudio_msg_header *header = (struct aaudio_msg_header *) msg->data; + if (msg->size < sizeof(struct aaudio_msg_header)) { + pr_err("aaudio: Msg size smaller than header (%lx)", msg->size); + return; + } + if (header->type == AAUDIO_MSG_TYPE_RESPONSE) { + aaudio_handle_reply(&a->bcem, msg); + } else if (header->type == AAUDIO_MSG_TYPE_COMMAND) { + aaudio_handle_command(a, msg); + } else if (header->type == AAUDIO_MSG_TYPE_NOTIFICATION) { + aaudio_handle_notification(a, msg); + } +} + +void aaudio_bce_in_queue_submit_pending(struct aaudio_bce_queue *q, size_t count) +{ + struct bce_qe_submission *s; + while (count--) { + if (bce_reserve_submission(q->sq, NULL)) { + pr_err("aaudio: Failed to reserve an event queue submission\n"); + break; + } + s = bce_next_submission(q->sq); + bce_set_submission_single(s, q->dma_addr + (dma_addr_t) (q->data_tail * q->el_size), q->el_size); + q->data_tail = (q->data_tail + 1) % q->el_count; + } + bce_submit_to_device(q->sq); +} + +struct aaudio_msg aaudio_reply_alloc(void) +{ + struct aaudio_msg ret; + ret.size = AAUDIO_BCE_QUEUE_ELEMENT_SIZE; + ret.data = kmalloc(ret.size, GFP_KERNEL); + return ret; +} + +void aaudio_reply_free(struct aaudio_msg *reply) +{ + kfree(reply->data); +} diff --git a/drivers/staging/apple-bce/audio/protocol_bce.h b/drivers/staging/apple-bce/audio/protocol_bce.h new file mode 100644 index 000000000000..14d26c05ddf9 --- /dev/null +++ b/drivers/staging/apple-bce/audio/protocol_bce.h @@ -0,0 +1,72 @@ +#ifndef AAUDIO_PROTOCOL_BCE_H +#define AAUDIO_PROTOCOL_BCE_H + +#include "protocol.h" +#include "../queue.h" + +#define AAUDIO_BCE_QUEUE_ELEMENT_SIZE 0x1000 +#define AAUDIO_BCE_QUEUE_ELEMENT_COUNT 20 + +#define AAUDIO_BCE_QUEUE_TAG_COUNT 1000 + +struct aaudio_device; + +struct aaudio_bce_queue_entry { + struct aaudio_msg *msg; + struct completion *cmpl; +}; +struct aaudio_bce_queue { + struct bce_queue_cq *cq; + struct bce_queue_sq *sq; + void *data; + dma_addr_t dma_addr; + size_t data_head, data_tail; + size_t el_size, el_count; +}; +struct aaudio_bce { + struct bce_queue_cq *cq; + struct aaudio_bce_queue qin; + struct aaudio_bce_queue qout; + int tag_num; + struct aaudio_bce_queue_entry *pending_entries[AAUDIO_BCE_QUEUE_TAG_COUNT]; + struct spinlock spinlock; +}; + +struct aaudio_send_ctx { + int status; + int tag_n; + unsigned long irq_flags; + struct aaudio_msg msg; + unsigned long timeout; +}; + +int aaudio_bce_init(struct aaudio_device *dev); +int __aaudio_send_prepare(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, char *tag); +void __aaudio_send(struct aaudio_bce *b, struct aaudio_send_ctx *ctx); +int __aaudio_send_cmd_sync(struct aaudio_bce *b, struct aaudio_send_ctx *ctx, struct aaudio_msg *reply); + +#define aaudio_send_with_tag(a, ctx, tag, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __aaudio_send_prepare(&(a)->bcem, (ctx), (tag)); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + __aaudio_send(&(a)->bcem, (ctx)); \ + } \ + (ctx)->status; \ +}) +#define aaudio_send(a, ctx, tout, fn, ...) aaudio_send_with_tag(a, ctx, NULL, tout, fn, ##__VA_ARGS__) + +#define aaudio_send_cmd_sync(a, ctx, reply, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __aaudio_send_prepare(&(a)->bcem, (ctx), NULL); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + (ctx)->status = __aaudio_send_cmd_sync(&(a)->bcem, (ctx), (reply)); \ + } \ + (ctx)->status; \ +}) + +struct aaudio_msg aaudio_reply_alloc(void); +void aaudio_reply_free(struct aaudio_msg *reply); + +#endif //AAUDIO_PROTOCOL_BCE_H diff --git a/drivers/staging/apple-bce/mailbox.c b/drivers/staging/apple-bce/mailbox.c new file mode 100644 index 000000000000..af1a59cb6170 --- /dev/null +++ b/drivers/staging/apple-bce/mailbox.c @@ -0,0 +1,151 @@ +#include "mailbox.h" +#include +#include "apple_bce.h" + +#define REG_MBOX_OUT_BASE 0x820 +#define REG_MBOX_REPLY_COUNTER 0x108 +#define REG_MBOX_REPLY_BASE 0x810 +#define REG_TIMESTAMP_BASE 0xC000 + +#define BCE_MBOX_TIMEOUT_MS 200 + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb) +{ + mb->reg_mb = reg_mb; + init_completion(&mb->mb_completion); +} + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv) +{ + u32 __iomem *regb; + + if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { + return -EEXIST; // We don't support two messages at once + } + reinit_completion(&mb->mb_completion); + + pr_debug("bce_mailbox_send: %llx\n", msg); + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); + iowrite32((u32) msg, regb); + iowrite32((u32) (msg >> 32), regb + 1); + iowrite32(0, regb + 2); + iowrite32(0, regb + 3); + + wait_for_completion_timeout(&mb->mb_completion, msecs_to_jiffies(BCE_MBOX_TIMEOUT_MS)); + if (atomic_read(&mb->mb_status) != 2) { // Didn't get the reply + atomic_set(&mb->mb_status, 0); + return -ETIMEDOUT; + } + + *recv = mb->mb_result; + pr_debug("bce_mailbox_send: reply %llx\n", *recv); + + atomic_set(&mb->mb_status, 0); + return 0; +} + +static int bce_mailbox_retrive_response(struct bce_mailbox *mb) +{ + u32 __iomem *regb; + u32 lo, hi; + int count, counter; + u32 res = ioread32((u8*) mb->reg_mb + REG_MBOX_REPLY_COUNTER); + count = (res >> 20) & 0xf; + counter = count; + pr_debug("bce_mailbox_retrive_response count=%i\n", count); + while (counter--) { + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_REPLY_BASE); + lo = ioread32(regb); + hi = ioread32(regb + 1); + ioread32(regb + 2); + ioread32(regb + 3); + pr_debug("bce_mailbox_retrive_response %llx\n", ((u64) hi << 32) | lo); + mb->mb_result = ((u64) hi << 32) | lo; + } + return count > 0 ? 0 : -ENODATA; +} + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb) +{ + int status = bce_mailbox_retrive_response(mb); + if (!status) { + atomic_set(&mb->mb_status, 2); + complete(&mb->mb_completion); + } + return status; +} + +static void bc_send_timestamp(struct timer_list *tl); + +void bce_timestamp_init(struct bce_timestamp *ts, void __iomem *reg) +{ + u32 __iomem *regb; + + spin_lock_init(&ts->stop_sl); + ts->stopped = false; + + ts->reg = reg; + + regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + + ioread32(regb); + mb(); + + timer_setup(&ts->timer, bc_send_timestamp, 0); +} + +void bce_timestamp_start(struct bce_timestamp *ts, bool is_initial) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + + if (is_initial) { + iowrite32((u32) -4, regb + 2); + iowrite32((u32) -1, regb); + } else { + iowrite32((u32) -3, regb + 2); + iowrite32((u32) -1, regb); + } + + spin_lock_irqsave(&ts->stop_sl, flags); + ts->stopped = false; + spin_unlock_irqrestore(&ts->stop_sl, flags); + mod_timer(&ts->timer, jiffies + msecs_to_jiffies(150)); +} + +void bce_timestamp_stop(struct bce_timestamp *ts) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + + spin_lock_irqsave(&ts->stop_sl, flags); + ts->stopped = true; + spin_unlock_irqrestore(&ts->stop_sl, flags); + timer_delete_sync(&ts->timer); + + iowrite32((u32) -2, regb + 2); + iowrite32((u32) -1, regb); +} + +static void bc_send_timestamp(struct timer_list *tl) +{ + struct bce_timestamp *ts; + unsigned long flags; + u32 __iomem *regb; + ktime_t bt; + + ts = container_of(tl, struct bce_timestamp, timer); + regb = (u32*) ((u8*) ts->reg + REG_TIMESTAMP_BASE); + local_irq_save(flags); + ioread32(regb + 2); + mb(); + bt = ktime_get_boottime(); + iowrite32((u32) bt, regb + 2); + iowrite32((u32) (bt >> 32), regb); + + spin_lock(&ts->stop_sl); + if (!ts->stopped) + mod_timer(&ts->timer, jiffies + msecs_to_jiffies(150)); + spin_unlock(&ts->stop_sl); + local_irq_restore(flags); +} diff --git a/drivers/staging/apple-bce/mailbox.h b/drivers/staging/apple-bce/mailbox.h new file mode 100644 index 000000000000..f3323f95ba51 --- /dev/null +++ b/drivers/staging/apple-bce/mailbox.h @@ -0,0 +1,53 @@ +#ifndef BCE_MAILBOX_H +#define BCE_MAILBOX_H + +#include +#include +#include + +struct bce_mailbox { + void __iomem *reg_mb; + + atomic_t mb_status; // possible statuses: 0 (no msg), 1 (has active msg), 2 (got reply) + struct completion mb_completion; + uint64_t mb_result; +}; + +enum bce_message_type { + BCE_MB_REGISTER_COMMAND_SQ = 0x7, // to-device + BCE_MB_REGISTER_COMMAND_CQ = 0x8, // to-device + BCE_MB_REGISTER_COMMAND_QUEUE_REPLY = 0xB, // to-host + BCE_MB_SET_FW_PROTOCOL_VERSION = 0xC, // both + BCE_MB_SLEEP_NO_STATE = 0x14, // to-device + BCE_MB_RESTORE_NO_STATE = 0x15, // to-device + BCE_MB_SAVE_STATE_AND_SLEEP = 0x17, // to-device + BCE_MB_RESTORE_STATE_AND_WAKE = 0x18, // to-device + BCE_MB_SAVE_STATE_AND_SLEEP_FAILURE = 0x19, // from-device + BCE_MB_SAVE_RESTORE_STATE_COMPLETE = 0x1A, // from-device +}; + +#define BCE_MB_MSG(type, value) (((u64) (type) << 58) | ((value) & 0x3FFFFFFFFFFFFFFLL)) +#define BCE_MB_TYPE(v) ((u32) (v >> 58)) +#define BCE_MB_VALUE(v) (v & 0x3FFFFFFFFFFFFFFLL) + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb); + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv); + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb); + + +struct bce_timestamp { + void __iomem *reg; + struct timer_list timer; + struct spinlock stop_sl; + bool stopped; +}; + +void bce_timestamp_init(struct bce_timestamp *ts, void __iomem *reg); + +void bce_timestamp_start(struct bce_timestamp *ts, bool is_initial); + +void bce_timestamp_stop(struct bce_timestamp *ts); + +#endif //BCEDRIVER_MAILBOX_H diff --git a/drivers/staging/apple-bce/queue.c b/drivers/staging/apple-bce/queue.c new file mode 100644 index 000000000000..bc9cd3bc6f0c --- /dev/null +++ b/drivers/staging/apple-bce/queue.c @@ -0,0 +1,390 @@ +#include "queue.h" +#include "apple_bce.h" + +#define REG_DOORBELL_BASE 0x44000 + +struct bce_queue_cq *bce_alloc_cq(struct apple_bce_device *dev, int qid, u32 el_count) +{ + struct bce_queue_cq *q; + q = kzalloc(sizeof(struct bce_queue_cq), GFP_KERNEL); + q->qid = qid; + q->type = BCE_QUEUE_CQ; + q->el_count = el_count; + q->data = dma_alloc_coherent(&dev->pci->dev, el_count * sizeof(struct bce_qe_completion), + &q->dma_handle, GFP_KERNEL); + if (!q->data) { + pr_err("DMA queue memory alloc failed\n"); + kfree(q); + return NULL; + } + return q; +} + +void bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) cq->qid; + cfg->el_count = (u16) cq->el_count; + cfg->vector_or_cq = 0; + cfg->_pad = 0; + cfg->addr = cq->dma_handle; + cfg->length = cq->el_count * sizeof(struct bce_qe_completion); +} + +void bce_free_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq) +{ + dma_free_coherent(&dev->pci->dev, cq->el_count * sizeof(struct bce_qe_completion), cq->data, cq->dma_handle); + kfree(cq); +} + +static void bce_handle_cq_completion(struct apple_bce_device *dev, struct bce_qe_completion *e, size_t *ce) +{ + struct bce_queue *target; + struct bce_queue_sq *target_sq; + struct bce_sq_completion_data *cmpl; + if (e->qid >= BCE_MAX_QUEUE_COUNT) { + pr_err("Device sent a response for qid (%u) >= BCE_MAX_QUEUE_COUNT\n", e->qid); + return; + } + target = dev->queues[e->qid]; + if (!target || target->type != BCE_QUEUE_SQ) { + pr_err("Device sent a response for qid (%u), which does not exist\n", e->qid); + return; + } + target_sq = (struct bce_queue_sq *) target; + if (target_sq->completion_tail != e->completion_index) { + pr_err("Completion index mismatch; this is likely going to make this driver unusable\n"); + return; + } + if (!target_sq->has_pending_completions) { + target_sq->has_pending_completions = true; + dev->int_sq_list[(*ce)++] = target_sq; + } + cmpl = &target_sq->completion_data[e->completion_index]; + cmpl->status = e->status; + cmpl->data_size = e->data_size; + cmpl->result = e->result; + wmb(); + target_sq->completion_tail = (target_sq->completion_tail + 1) % target_sq->el_count; +} + +void bce_handle_cq_completions(struct apple_bce_device *dev, struct bce_queue_cq *cq) +{ + size_t ce = 0; + struct bce_qe_completion *e; + struct bce_queue_sq *sq; + e = bce_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + return; + mb(); + while (true) { + e = bce_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + break; + // pr_info("apple-bce: compl: %i: %i %llx %llx", e->qid, e->status, e->data_size, e->result); + bce_handle_cq_completion(dev, e, &ce); + e->flags = 0; + cq->index = (cq->index + 1) % cq->el_count; + } + mb(); + iowrite32(cq->index, (u32 *) ((u8 *) dev->reg_mem_dma + REG_DOORBELL_BASE) + cq->qid); + while (ce) { + --ce; + sq = dev->int_sq_list[ce]; + sq->completion(sq); + sq->has_pending_completions = false; + } +} + + +struct bce_queue_sq *bce_alloc_sq(struct apple_bce_device *dev, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *q; + q = kzalloc(sizeof(struct bce_queue_sq), GFP_KERNEL); + q->qid = qid; + q->type = BCE_QUEUE_SQ; + q->el_size = el_size; + q->el_count = el_count; + q->data = dma_alloc_coherent(&dev->pci->dev, el_count * el_size, + &q->dma_handle, GFP_KERNEL); + q->completion = compl; + q->userdata = userdata; + q->completion_data = kzalloc(sizeof(struct bce_sq_completion_data) * el_count, GFP_KERNEL); + q->reg_mem_dma = dev->reg_mem_dma; + atomic_set(&q->available_commands, el_count - 1); + init_completion(&q->available_command_completion); + atomic_set(&q->available_command_completion_waiting_count, 0); + if (!q->data) { + pr_err("DMA queue memory alloc failed\n"); + kfree(q); + return NULL; + } + return q; +} + +void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) sq->qid; + cfg->el_count = (u16) sq->el_count; + cfg->vector_or_cq = (u16) cq->qid; + cfg->_pad = 0; + cfg->addr = sq->dma_handle; + cfg->length = sq->el_count * sq->el_size; +} + +void bce_free_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq) +{ + dma_free_coherent(&dev->pci->dev, sq->el_count * sq->el_size, sq->data, sq->dma_handle); + kfree(sq); +} + +int bce_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout) +{ + while (atomic_dec_if_positive(&sq->available_commands) < 0) { + if (!timeout || !*timeout) + return -EAGAIN; + atomic_inc(&sq->available_command_completion_waiting_count); + *timeout = wait_for_completion_timeout(&sq->available_command_completion, *timeout); + if (!*timeout) { + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) < 0) + try_wait_for_completion(&sq->available_command_completion); /* consume the pending completion */ + } + } + return 0; +} + +void bce_cancel_submission_reservation(struct bce_queue_sq *sq) +{ + atomic_inc(&sq->available_commands); +} + +void *bce_next_submission(struct bce_queue_sq *sq) +{ + void *ret = bce_sq_element(sq, sq->tail); + sq->tail = (sq->tail + 1) % sq->el_count; + return ret; +} + +void bce_submit_to_device(struct bce_queue_sq *sq) +{ + mb(); + iowrite32(sq->tail, (u32 *) ((u8 *) sq->reg_mem_dma + REG_DOORBELL_BASE) + sq->qid); +} + +void bce_notify_submission_complete(struct bce_queue_sq *sq) +{ + sq->head = (sq->head + 1) % sq->el_count; + atomic_inc(&sq->available_commands); + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) >= 0) { + complete(&sq->available_command_completion); + } +} + +void bce_set_submission_single(struct bce_qe_submission *element, dma_addr_t addr, size_t size) +{ + element->addr = addr; + element->length = size; + element->segl_addr = element->segl_length = 0; +} + +static void bce_cmdq_completion(struct bce_queue_sq *q); + +struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_device *dev, int qid, u32 el_count) +{ + struct bce_queue_cmdq *q; + q = kzalloc(sizeof(struct bce_queue_cmdq), GFP_KERNEL); + q->sq = bce_alloc_sq(dev, qid, BCE_CMD_SIZE, el_count, bce_cmdq_completion, q); + if (!q->sq) { + kfree(q); + return NULL; + } + spin_lock_init(&q->lck); + q->tres = kzalloc(sizeof(struct bce_queue_cmdq_result_el*) * el_count, GFP_KERNEL); + if (!q->tres) { + kfree(q); + return NULL; + } + return q; +} + +void bce_free_cmdq(struct apple_bce_device *dev, struct bce_queue_cmdq *cmdq) +{ + bce_free_sq(dev, cmdq->sq); + kfree(cmdq->tres); + kfree(cmdq); +} + +void bce_cmdq_completion(struct bce_queue_sq *q) +{ + struct bce_queue_cmdq_result_el *el; + struct bce_queue_cmdq *cmdq = q->userdata; + struct bce_sq_completion_data *result; + + spin_lock(&cmdq->lck); + while ((result = bce_next_completion(q))) { + el = cmdq->tres[cmdq->sq->head]; + if (el) { + el->result = result->result; + el->status = result->status; + mb(); + complete(&el->cmpl); + } else { + pr_err("apple-bce: Unexpected command queue completion\n"); + } + cmdq->tres[cmdq->sq->head] = NULL; + bce_notify_submission_complete(q); + } + spin_unlock(&cmdq->lck); +} + +static __always_inline void *bce_cmd_start(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + void *ret; + unsigned long timeout; + init_completion(&res->cmpl); + mb(); + + timeout = msecs_to_jiffies(1000L * 60 * 5); /* wait for up to ~5 minutes */ + if (bce_reserve_submission(cmdq->sq, &timeout)) + return NULL; + + spin_lock(&cmdq->lck); + cmdq->tres[cmdq->sq->tail] = res; + ret = bce_next_submission(cmdq->sq); + return ret; +} + +static __always_inline void bce_cmd_finish(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + bce_submit_to_device(cmdq->sq); + spin_unlock(&cmdq->lck); + + wait_for_completion(&res->cmpl); + mb(); +} + +u32 bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_register_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_REGISTER_MEMORY_QUEUE; + cmd->flags = (u16) ((name ? 2 : 0) | (isdirout ? 1 : 0)); + cmd->qid = cfg->qid; + cmd->el_count = cfg->el_count; + cmd->vector_or_cq = cfg->vector_or_cq; + memset(cmd->name, 0, sizeof(cmd->name)); + if (name) { + cmd->name_len = (u16) min(strlen(name), (size_t) sizeof(cmd->name)); + memcpy(cmd->name, name, cmd->name_len); + } else { + cmd->name_len = 0; + } + cmd->addr = cfg->addr; + cmd->length = cfg->length; + + bce_cmd_finish(cmdq, &res); + return res.status; +} + +u32 bce_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_UNREGISTER_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + bce_cmd_finish(cmdq, &res); + return res.status; +} + +u32 bce_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = bce_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_FLUSH_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + bce_cmd_finish(cmdq, &res); + return res.status; +} + + +struct bce_queue_cq *bce_create_cq(struct apple_bce_device *dev, u32 el_count) +{ + struct bce_queue_cq *cq; + struct bce_queue_memcfg cfg; + int qid = ida_simple_get(&dev->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); + if (qid < 0) + return NULL; + cq = bce_alloc_cq(dev, qid, el_count); + if (!cq) + return NULL; + bce_get_cq_memcfg(cq, &cfg); + if (bce_cmd_register_queue(dev->cmd_cmdq, &cfg, NULL, false) != 0) { + pr_err("apple-bce: CQ registration failed (%i)", qid); + bce_free_cq(dev, cq); + ida_simple_remove(&dev->queue_ida, (uint) qid); + return NULL; + } + dev->queues[qid] = (struct bce_queue *) cq; + return cq; +} + +struct bce_queue_sq *bce_create_sq(struct apple_bce_device *dev, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *sq; + struct bce_queue_memcfg cfg; + int qid; + if (cq == NULL) + return NULL; /* cq can not be null */ + if (name == NULL) + return NULL; /* name can not be null */ + if (direction != DMA_TO_DEVICE && direction != DMA_FROM_DEVICE) + return NULL; /* unsupported direction */ + qid = ida_simple_get(&dev->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); + if (qid < 0) + return NULL; + sq = bce_alloc_sq(dev, qid, sizeof(struct bce_qe_submission), el_count, compl, userdata); + if (!sq) + return NULL; + bce_get_sq_memcfg(sq, cq, &cfg); + if (bce_cmd_register_queue(dev->cmd_cmdq, &cfg, name, direction != DMA_FROM_DEVICE) != 0) { + pr_err("apple-bce: SQ registration failed (%i)", qid); + bce_free_sq(dev, sq); + ida_simple_remove(&dev->queue_ida, (uint) qid); + return NULL; + } + spin_lock(&dev->queues_lock); + dev->queues[qid] = (struct bce_queue *) sq; + spin_unlock(&dev->queues_lock); + return sq; +} + +void bce_destroy_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq) +{ + if (!dev->is_being_removed && bce_cmd_unregister_memory_queue(dev->cmd_cmdq, (u16) cq->qid)) + pr_err("apple-bce: CQ unregister failed"); + spin_lock(&dev->queues_lock); + dev->queues[cq->qid] = NULL; + spin_unlock(&dev->queues_lock); + ida_simple_remove(&dev->queue_ida, (uint) cq->qid); + bce_free_cq(dev, cq); +} + +void bce_destroy_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq) +{ + if (!dev->is_being_removed && bce_cmd_unregister_memory_queue(dev->cmd_cmdq, (u16) sq->qid)) + pr_err("apple-bce: CQ unregister failed"); + spin_lock(&dev->queues_lock); + dev->queues[sq->qid] = NULL; + spin_unlock(&dev->queues_lock); + ida_simple_remove(&dev->queue_ida, (uint) sq->qid); + bce_free_sq(dev, sq); +} \ No newline at end of file diff --git a/drivers/staging/apple-bce/queue.h b/drivers/staging/apple-bce/queue.h new file mode 100644 index 000000000000..8368ac5dfca8 --- /dev/null +++ b/drivers/staging/apple-bce/queue.h @@ -0,0 +1,177 @@ +#ifndef BCE_QUEUE_H +#define BCE_QUEUE_H + +#include +#include + +#define BCE_CMD_SIZE 0x40 + +struct apple_bce_device; + +enum bce_queue_type { + BCE_QUEUE_CQ, BCE_QUEUE_SQ +}; +struct bce_queue { + int qid; + int type; +}; +struct bce_queue_cq { + int qid; + int type; + u32 el_count; + dma_addr_t dma_handle; + void *data; + + u32 index; +}; +struct bce_queue_sq; +typedef void (*bce_sq_completion)(struct bce_queue_sq *q); +struct bce_sq_completion_data { + u32 status; + u64 data_size; + u64 result; +}; +struct bce_queue_sq { + int qid; + int type; + u32 el_size; + u32 el_count; + dma_addr_t dma_handle; + void *data; + void *userdata; + void __iomem *reg_mem_dma; + + atomic_t available_commands; + struct completion available_command_completion; + atomic_t available_command_completion_waiting_count; + u32 head, tail; + + u32 completion_cidx, completion_tail; + struct bce_sq_completion_data *completion_data; + bool has_pending_completions; + bce_sq_completion completion; +}; + +struct bce_queue_cmdq_result_el { + struct completion cmpl; + u32 status; + u64 result; +}; +struct bce_queue_cmdq { + struct bce_queue_sq *sq; + struct spinlock lck; + struct bce_queue_cmdq_result_el **tres; +}; + +struct bce_queue_memcfg { + u16 qid; + u16 el_count; + u16 vector_or_cq; + u16 _pad; + u64 addr; + u64 length; +}; + +enum bce_qe_completion_status { + BCE_COMPLETION_SUCCESS = 0, + BCE_COMPLETION_ERROR = 1, + BCE_COMPLETION_ABORTED = 2, + BCE_COMPLETION_NO_SPACE = 3, + BCE_COMPLETION_OVERRUN = 4 +}; +enum bce_qe_completion_flags { + BCE_COMPLETION_FLAG_PENDING = 0x8000 +}; +struct bce_qe_completion { + u64 result; + u64 data_size; + u16 qid; + u16 completion_index; + u16 status; // bce_qe_completion_status + u16 flags; // bce_qe_completion_flags +}; + +struct bce_qe_submission { + u64 length; + u64 addr; + + u64 segl_addr; + u64 segl_length; +}; + +enum bce_cmdq_command { + BCE_CMD_REGISTER_MEMORY_QUEUE = 0x20, + BCE_CMD_UNREGISTER_MEMORY_QUEUE = 0x30, + BCE_CMD_FLUSH_MEMORY_QUEUE = 0x40, + BCE_CMD_SET_MEMORY_QUEUE_PROPERTY = 0x50 +}; +struct bce_cmdq_simple_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; +}; +struct bce_cmdq_register_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; + u16 _pad; + u16 el_count; + u16 vector_or_cq; + u16 _pad2; + u16 name_len; + char name[0x20]; + u64 addr; + u64 length; +}; + +static __always_inline void *bce_sq_element(struct bce_queue_sq *q, int i) { + return (void *) ((u8 *) q->data + q->el_size * i); +} +static __always_inline void *bce_cq_element(struct bce_queue_cq *q, int i) { + return (void *) ((struct bce_qe_completion *) q->data + i); +} + +static __always_inline struct bce_sq_completion_data *bce_next_completion(struct bce_queue_sq *sq) { + struct bce_sq_completion_data *res; + rmb(); + if (sq->completion_cidx == sq->completion_tail) + return NULL; + res = &sq->completion_data[sq->completion_cidx]; + sq->completion_cidx = (sq->completion_cidx + 1) % sq->el_count; + return res; +} + +struct bce_queue_cq *bce_alloc_cq(struct apple_bce_device *dev, int qid, u32 el_count); +void bce_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void bce_free_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq); +void bce_handle_cq_completions(struct apple_bce_device *dev, struct bce_queue_cq *cq); + +struct bce_queue_sq *bce_alloc_sq(struct apple_bce_device *dev, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata); +void bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void bce_free_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq); +int bce_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout); +void bce_cancel_submission_reservation(struct bce_queue_sq *sq); +void *bce_next_submission(struct bce_queue_sq *sq); +void bce_submit_to_device(struct bce_queue_sq *sq); +void bce_notify_submission_complete(struct bce_queue_sq *sq); + +void bce_set_submission_single(struct bce_qe_submission *element, dma_addr_t addr, size_t size); + +struct bce_queue_cmdq *bce_alloc_cmdq(struct apple_bce_device *dev, int qid, u32 el_count); +void bce_free_cmdq(struct apple_bce_device *dev, struct bce_queue_cmdq *cmdq); + +u32 bce_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout); +u32 bce_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); +u32 bce_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); + + +/* User API - Creates and registers the queue */ + +struct bce_queue_cq *bce_create_cq(struct apple_bce_device *dev, u32 el_count); +struct bce_queue_sq *bce_create_sq(struct apple_bce_device *dev, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata); +void bce_destroy_cq(struct apple_bce_device *dev, struct bce_queue_cq *cq); +void bce_destroy_sq(struct apple_bce_device *dev, struct bce_queue_sq *sq); + +#endif //BCEDRIVER_MAILBOX_H diff --git a/drivers/staging/apple-bce/queue_dma.c b/drivers/staging/apple-bce/queue_dma.c new file mode 100644 index 000000000000..b236613285c0 --- /dev/null +++ b/drivers/staging/apple-bce/queue_dma.c @@ -0,0 +1,220 @@ +#include "queue_dma.h" +#include +#include +#include "queue.h" + +static int bce_alloc_scatterlist_from_vm(struct sg_table *tbl, void *data, size_t len); +static struct bce_segment_list_element_hostinfo *bce_map_segment_list( + struct device *dev, struct scatterlist *pages, int pagen); +static void bce_unmap_segement_list(struct device *dev, struct bce_segment_list_element_hostinfo *list); + +int bce_map_dma_buffer(struct device *dev, struct bce_dma_buffer *buf, struct sg_table scatterlist, + enum dma_data_direction dir) +{ + int cnt; + + buf->direction = dir; + buf->scatterlist = scatterlist; + buf->seglist_hostinfo = NULL; + + cnt = dma_map_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); + if (cnt != buf->scatterlist.nents) { + pr_err("apple-bce: DMA scatter list mapping returned an unexpected count: %i\n", cnt); + dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); + return -EIO; + } + if (cnt == 1) + return 0; + + buf->seglist_hostinfo = bce_map_segment_list(dev, buf->scatterlist.sgl, buf->scatterlist.nents); + if (!buf->seglist_hostinfo) { + pr_err("apple-bce: Creating segment list failed\n"); + dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, dir); + return -EIO; + } + return 0; +} + +int bce_map_dma_buffer_vm(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir) +{ + int status; + struct sg_table scatterlist; + if ((status = bce_alloc_scatterlist_from_vm(&scatterlist, data, len))) + return status; + if ((status = bce_map_dma_buffer(dev, buf, scatterlist, dir))) { + sg_free_table(&scatterlist); + return status; + } + return 0; +} + +int bce_map_dma_buffer_km(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir) +{ + /* Kernel memory is continuous which is great for us. */ + int status; + struct sg_table scatterlist; + if ((status = sg_alloc_table(&scatterlist, 1, GFP_KERNEL))) { + sg_free_table(&scatterlist); + return status; + } + sg_set_buf(scatterlist.sgl, data, (uint) len); + if ((status = bce_map_dma_buffer(dev, buf, scatterlist, dir))) { + sg_free_table(&scatterlist); + return status; + } + return 0; +} + +void bce_unmap_dma_buffer(struct device *dev, struct bce_dma_buffer *buf) +{ + dma_unmap_sg(dev, buf->scatterlist.sgl, buf->scatterlist.nents, buf->direction); + bce_unmap_segement_list(dev, buf->seglist_hostinfo); +} + + +static int bce_alloc_scatterlist_from_vm(struct sg_table *tbl, void *data, size_t len) +{ + int status, i; + struct page **pages; + size_t off, start_page, end_page, page_count; + off = (size_t) data % PAGE_SIZE; + start_page = (size_t) data / PAGE_SIZE; + end_page = ((size_t) data + len - 1) / PAGE_SIZE; + page_count = end_page - start_page + 1; + + if (page_count > PAGE_SIZE / sizeof(struct page *)) + pages = vmalloc(page_count * sizeof(struct page *)); + else + pages = kmalloc(page_count * sizeof(struct page *), GFP_KERNEL); + + for (i = 0; i < page_count; i++) + pages[i] = vmalloc_to_page((void *) ((start_page + i) * PAGE_SIZE)); + + if ((status = sg_alloc_table_from_pages(tbl, pages, page_count, (unsigned int) off, len, GFP_KERNEL))) { + sg_free_table(tbl); + } + + if (page_count > PAGE_SIZE / sizeof(struct page *)) + vfree(pages); + else + kfree(pages); + return status; +} + +#define BCE_ELEMENTS_PER_PAGE ((PAGE_SIZE - sizeof(struct bce_segment_list_header)) \ + / sizeof(struct bce_segment_list_element)) +#define BCE_ELEMENTS_PER_ADDITIONAL_PAGE (PAGE_SIZE / sizeof(struct bce_segment_list_element)) + +static struct bce_segment_list_element_hostinfo *bce_map_segment_list( + struct device *dev, struct scatterlist *pages, int pagen) +{ + size_t ptr, pptr = 0; + struct bce_segment_list_header theader; /* a temp header, to store the initial seg */ + struct bce_segment_list_header *header; + struct bce_segment_list_element *el, *el_end; + struct bce_segment_list_element_hostinfo *out, *pout, *out_root; + struct scatterlist *sg; + int i; + header = &theader; + out = out_root = NULL; + el = el_end = NULL; + for_each_sg(pages, sg, pagen, i) { + if (el >= el_end) { + /* allocate a new page, this will be also done for the first element */ + ptr = __get_free_page(GFP_KERNEL); + if (pptr && ptr == pptr + PAGE_SIZE) { + out->page_count++; + header->element_count += BCE_ELEMENTS_PER_ADDITIONAL_PAGE; + el_end += BCE_ELEMENTS_PER_ADDITIONAL_PAGE; + } else { + header = (void *) ptr; + header->element_count = BCE_ELEMENTS_PER_PAGE; + header->data_size = 0; + header->next_segl_addr = 0; + header->next_segl_length = 0; + el = (void *) (header + 1); + el_end = el + BCE_ELEMENTS_PER_PAGE; + + if (out) { + out->next = kmalloc(sizeof(struct bce_segment_list_element_hostinfo), GFP_KERNEL); + out = out->next; + } else { + out_root = out = kmalloc(sizeof(struct bce_segment_list_element_hostinfo), GFP_KERNEL); + } + out->page_start = (void *) ptr; + out->page_count = 1; + out->dma_start = DMA_MAPPING_ERROR; + out->next = NULL; + } + pptr = ptr; + } + el->addr = sg->dma_address; + el->length = sg->length; + header->data_size += el->length; + } + + /* DMA map */ + out = out_root; + pout = NULL; + while (out) { + out->dma_start = dma_map_single(dev, out->page_start, out->page_count * PAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, out->dma_start)) + goto error; + if (pout) { + header = pout->page_start; + header->next_segl_addr = out->dma_start; + header->next_segl_length = out->page_count * PAGE_SIZE; + } + pout = out; + out = out->next; + } + return out_root; + + error: + bce_unmap_segement_list(dev, out_root); + return NULL; +} + +static void bce_unmap_segement_list(struct device *dev, struct bce_segment_list_element_hostinfo *list) +{ + struct bce_segment_list_element_hostinfo *next; + while (list) { + if (list->dma_start != DMA_MAPPING_ERROR) + dma_unmap_single(dev, list->dma_start, list->page_count * PAGE_SIZE, DMA_TO_DEVICE); + next = list->next; + kfree(list); + list = next; + } +} + +int bce_set_submission_buf(struct bce_qe_submission *element, struct bce_dma_buffer *buf, size_t offset, size_t length) +{ + struct bce_segment_list_element_hostinfo *seg; + struct bce_segment_list_header *seg_header; + + seg = buf->seglist_hostinfo; + if (!seg) { + element->addr = buf->scatterlist.sgl->dma_address + offset; + element->length = length; + element->segl_addr = 0; + element->segl_length = 0; + return 0; + } + + while (seg) { + seg_header = seg->page_start; + if (offset <= seg_header->data_size) + break; + offset -= seg_header->data_size; + seg = seg->next; + } + if (!seg) + return -EINVAL; + element->addr = offset; + element->length = buf->scatterlist.sgl->dma_length; + element->segl_addr = seg->dma_start; + element->segl_length = seg->page_count * PAGE_SIZE; + return 0; +} \ No newline at end of file diff --git a/drivers/staging/apple-bce/queue_dma.h b/drivers/staging/apple-bce/queue_dma.h new file mode 100644 index 000000000000..f8a57e50e7a3 --- /dev/null +++ b/drivers/staging/apple-bce/queue_dma.h @@ -0,0 +1,50 @@ +#ifndef BCE_QUEUE_DMA_H +#define BCE_QUEUE_DMA_H + +#include + +struct bce_qe_submission; + +struct bce_segment_list_header { + u64 element_count; + u64 data_size; + + u64 next_segl_addr; + u64 next_segl_length; +}; +struct bce_segment_list_element { + u64 addr; + u64 length; +}; + +struct bce_segment_list_element_hostinfo { + struct bce_segment_list_element_hostinfo *next; + void *page_start; + size_t page_count; + dma_addr_t dma_start; +}; + + +struct bce_dma_buffer { + enum dma_data_direction direction; + struct sg_table scatterlist; + struct bce_segment_list_element_hostinfo *seglist_hostinfo; +}; + +/* NOTE: Takes ownership of the sg_table if it succeeds. Ownership is not transferred on failure. */ +int bce_map_dma_buffer(struct device *dev, struct bce_dma_buffer *buf, struct sg_table scatterlist, + enum dma_data_direction dir); + +/* Creates a buffer from virtual memory (vmalloc) */ +int bce_map_dma_buffer_vm(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir); + +/* Creates a buffer from kernel memory (kmalloc) */ +int bce_map_dma_buffer_km(struct device *dev, struct bce_dma_buffer *buf, void *data, size_t len, + enum dma_data_direction dir); + +void bce_unmap_dma_buffer(struct device *dev, struct bce_dma_buffer *buf); + +int bce_set_submission_buf(struct bce_qe_submission *element, struct bce_dma_buffer *buf, size_t offset, size_t length); + +#endif //BCE_QUEUE_DMA_H diff --git a/drivers/staging/apple-bce/vhci/command.h b/drivers/staging/apple-bce/vhci/command.h new file mode 100644 index 000000000000..26619e0bccfa --- /dev/null +++ b/drivers/staging/apple-bce/vhci/command.h @@ -0,0 +1,204 @@ +#ifndef BCE_VHCI_COMMAND_H +#define BCE_VHCI_COMMAND_H + +#include "queue.h" +#include +#include + +#define BCE_VHCI_CMD_TIMEOUT_SHORT msecs_to_jiffies(2000) +#define BCE_VHCI_CMD_TIMEOUT_LONG msecs_to_jiffies(30000) + +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2 2 +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS (1 << BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2) + +typedef u8 bce_vhci_port_t; +typedef u8 bce_vhci_device_t; + +enum bce_vhci_command { + BCE_VHCI_CMD_CONTROLLER_ENABLE = 1, + BCE_VHCI_CMD_CONTROLLER_DISABLE = 2, + BCE_VHCI_CMD_CONTROLLER_START = 3, + BCE_VHCI_CMD_CONTROLLER_PAUSE = 4, + + BCE_VHCI_CMD_PORT_POWER_ON = 0x10, + BCE_VHCI_CMD_PORT_POWER_OFF = 0x11, + BCE_VHCI_CMD_PORT_RESUME = 0x12, + BCE_VHCI_CMD_PORT_SUSPEND = 0x13, + BCE_VHCI_CMD_PORT_RESET = 0x14, + BCE_VHCI_CMD_PORT_DISABLE = 0x15, + BCE_VHCI_CMD_PORT_STATUS = 0x16, + + BCE_VHCI_CMD_DEVICE_CREATE = 0x30, + BCE_VHCI_CMD_DEVICE_DESTROY = 0x31, + + BCE_VHCI_CMD_ENDPOINT_CREATE = 0x40, + BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x41, + BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x42, + BCE_VHCI_CMD_ENDPOINT_RESET = 0x44, + + /* Device to host only */ + BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE = 0x43, + BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000, + BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS = 0x1005 +}; + +enum bce_vhci_endpoint_state { + BCE_VHCI_ENDPOINT_ACTIVE = 0, + BCE_VHCI_ENDPOINT_PAUSED = 1, + BCE_VHCI_ENDPOINT_STALLED = 2 +}; + +static inline int bce_vhci_cmd_controller_enable(struct bce_vhci_command_queue *q, u8 busNum, u16 *portMask) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE; + cmd.param1 = 0x7100u | busNum; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); + if (!status) + *portMask = (u16) res.param2; + return status; +} +static inline int bce_vhci_cmd_controller_disable(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_start(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_pause(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_PAUSE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_port_power_on(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_power_off(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_OFF; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_resume(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESUME; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_suspend(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_SUSPEND; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_reset(struct bce_vhci_command_queue *q, bce_vhci_port_t port, u32 timeout) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESET; + cmd.param1 = port; + cmd.param2 = timeout; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_disable(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_DISABLE; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_status(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + u32 clearFlags, u32 *resStatus) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; + cmd.param1 = port; + cmd.param2 = clearFlags & 0x560000; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status >= 0) + *resStatus = (u32) res.param2; + return status; +} + +static inline int bce_vhci_cmd_device_create(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + bce_vhci_device_t *dev) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE; + cmd.param1 = port; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (!status) + *dev = (bce_vhci_device_t) res.param2; + return status; +} +static inline int bce_vhci_cmd_device_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = dev; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_endpoint_create(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, + struct usb_endpoint_descriptor *desc) +{ + struct bce_vhci_message cmd, res; + int endpoint_type = usb_endpoint_type(desc); + int maxp = usb_endpoint_maxp(desc); + int maxp_burst = usb_endpoint_maxp_mult(desc) * maxp; + u8 max_active_requests_pow2 = 0; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE; + cmd.param1 = dev | ((desc->bEndpointAddress & 0x8Fu) << 8); + if (endpoint_type == USB_ENDPOINT_XFER_BULK) + max_active_requests_pow2 = BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2; + cmd.param2 = endpoint_type | ((max_active_requests_pow2 & 0xf) << 4) | (maxp << 16) | ((u64) maxp_burst << 32); + if (endpoint_type == USB_ENDPOINT_XFER_INT) + cmd.param2 |= (desc->bInterval - 1) << 8; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_set_state(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint, + enum bce_vhci_endpoint_state newState, enum bce_vhci_endpoint_state *retState) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_SET_STATE; + cmd.param1 = dev | (endpoint << 8); + cmd.param2 = (u64) newState; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status != BCE_VHCI_INTERNAL_ERROR && status != BCE_VHCI_NO_POWER) + *retState = (enum bce_vhci_endpoint_state) res.param2; + return status; +} +static inline int bce_vhci_cmd_endpoint_reset(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} + + +#endif //BCE_VHCI_COMMAND_H diff --git a/drivers/staging/apple-bce/vhci/queue.c b/drivers/staging/apple-bce/vhci/queue.c new file mode 100644 index 000000000000..7b0b5027157b --- /dev/null +++ b/drivers/staging/apple-bce/vhci/queue.c @@ -0,0 +1,268 @@ +#include "queue.h" +#include "vhci.h" +#include "../apple_bce.h" + + +static void bce_vhci_message_queue_completion(struct bce_queue_sq *sq); + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name) +{ + int status; + ret->cq = bce_create_cq(vhci->dev, VHCI_EVENT_QUEUE_EL_COUNT); + if (!ret->cq) + return -EINVAL; + ret->sq = bce_create_sq(vhci->dev, ret->cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_TO_DEVICE, + bce_vhci_message_queue_completion, ret); + if (!ret->sq) { + status = -EINVAL; + goto fail_cq; + } + ret->data = dma_alloc_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + status = -EINVAL; + goto fail_sq; + } + return 0; + +fail_sq: + bce_destroy_sq(vhci->dev, ret->sq); + ret->sq = NULL; +fail_cq: + bce_destroy_cq(vhci->dev, ret->cq); + ret->cq = NULL; + return status; +} + +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q) +{ + if (!q->cq) + return; + dma_free_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + bce_destroy_sq(vhci->dev, q->sq); + bce_destroy_cq(vhci->dev, q->cq); +} + +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req) +{ + int sidx; + struct bce_qe_submission *s; + sidx = q->sq->tail; + s = bce_next_submission(q->sq); + pr_debug("bce-vhci: Send message: %x s=%x p1=%x p2=%llx\n", req->cmd, req->status, req->param1, req->param2); + q->data[sidx] = *req; + bce_set_submission_single(s, q->dma_addr + sizeof(struct bce_vhci_message) * sidx, + sizeof(struct bce_vhci_message)); + bce_submit_to_device(q->sq); +} + +static void bce_vhci_message_queue_completion(struct bce_queue_sq *sq) +{ + while (bce_next_completion(sq)) + bce_notify_submission_complete(sq); +} + + + +static void bce_vhci_event_queue_completion(struct bce_queue_sq *sq); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_sq_completion compl) +{ + ret->vhci = vhci; + + ret->sq = bce_create_sq(vhci->dev, vhci->ev_cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_FROM_DEVICE, compl, ret); + if (!ret->sq) + return -EINVAL; + ret->data = dma_alloc_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + bce_destroy_sq(vhci->dev, ret->sq); + ret->sq = NULL; + return -EINVAL; + } + + init_completion(&ret->queue_empty_completion); + bce_vhci_event_queue_submit_pending(ret, VHCI_EVENT_PENDING_COUNT); + return 0; +} + +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb) +{ + ret->cb = cb; + return __bce_vhci_event_queue_create(vhci, ret, name, bce_vhci_event_queue_completion); +} + +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q) +{ + if (!q->sq) + return; + dma_free_coherent(&vhci->dev->pci->dev, sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + bce_destroy_sq(vhci->dev, q->sq); +} + +static void bce_vhci_event_queue_completion(struct bce_queue_sq *sq) +{ + struct bce_sq_completion_data *cd; + struct bce_vhci_event_queue *ev = sq->userdata; + struct bce_vhci_message *msg; + size_t cnt = 0; + + while ((cd = bce_next_completion(sq))) { + if (cd->status == BCE_COMPLETION_ABORTED) { /* We flushed the queue */ + bce_notify_submission_complete(sq); + continue; + } + msg = &ev->data[sq->head]; + pr_debug("bce-vhci: Got event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + ev->cb(ev, msg); + + bce_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(ev, cnt); + if (atomic_read(&sq->available_commands) == sq->el_count - 1) + complete(&ev->queue_empty_completion); +} + +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count) +{ + int idx; + struct bce_qe_submission *s; + while (count--) { + if (bce_reserve_submission(q->sq, NULL)) { + pr_err("bce-vhci: Failed to reserve an event queue submission\n"); + break; + } + idx = q->sq->tail; + s = bce_next_submission(q->sq); + bce_set_submission_single(s, + q->dma_addr + idx * sizeof(struct bce_vhci_message), sizeof(struct bce_vhci_message)); + } + bce_submit_to_device(q->sq); +} + +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q) +{ + unsigned long timeout; + reinit_completion(&q->queue_empty_completion); + if (bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, q->sq->qid)) + pr_warn("bce-vhci: failed to flush event queue\n"); + timeout = msecs_to_jiffies(5000); + while (atomic_read(&q->sq->available_commands) != q->sq->el_count - 1) { + timeout = wait_for_completion_timeout(&q->queue_empty_completion, timeout); + if (timeout == 0) { + pr_err("bce-vhci: waiting for queue to be flushed timed out\n"); + break; + } + } +} + +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q) +{ + if (atomic_read(&q->sq->available_commands) != q->sq->el_count - 1) { + pr_err("bce-vhci: resume of a queue with pending submissions\n"); + return; + } + bce_vhci_event_queue_submit_pending(q, VHCI_EVENT_PENDING_COUNT); +} + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq) +{ + ret->mq = mq; + ret->completion.result = NULL; + init_completion(&ret->completion.completion); + spin_lock_init(&ret->completion_lock); + mutex_init(&ret->mutex); +} + +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq) +{ + spin_lock(&cq->completion_lock); + if (cq->completion.result) { + memset(cq->completion.result, 0, sizeof(struct bce_vhci_message)); + cq->completion.result->status = BCE_VHCI_ABORT; + complete(&cq->completion.completion); + cq->completion.result = NULL; + } + spin_unlock(&cq->completion_lock); + mutex_lock(&cq->mutex); + mutex_unlock(&cq->mutex); + mutex_destroy(&cq->mutex); +} + +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg) +{ + struct bce_vhci_command_queue_completion *c = &cq->completion; + + spin_lock(&cq->completion_lock); + if (c->result) { + *c->result = *msg; + complete(&c->completion); + c->result = NULL; + } + spin_unlock(&cq->completion_lock); +} + +static int __bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + struct bce_vhci_command_queue_completion *c; + struct bce_vhci_message creq; + c = &cq->completion; + + if ((status = bce_reserve_submission(cq->mq->sq, &timeout))) + return status; + + spin_lock(&cq->completion_lock); + c->result = res; + reinit_completion(&c->completion); + spin_unlock(&cq->completion_lock); + + bce_vhci_message_queue_write(cq->mq, req); + + if (!wait_for_completion_timeout(&c->completion, timeout)) { + /* we ran out of time, send cancellation */ + pr_debug("bce-vhci: command timed out req=%x\n", req->cmd); + if ((status = bce_reserve_submission(cq->mq->sq, &timeout))) + return status; + + creq = *req; + creq.cmd |= 0x4000; + bce_vhci_message_queue_write(cq->mq, &creq); + + if (!wait_for_completion_timeout(&c->completion, 1000)) { + pr_err("bce-vhci: Possible desync, cmd cancel timed out\n"); + + spin_lock(&cq->completion_lock); + c->result = NULL; + spin_unlock(&cq->completion_lock); + return -ETIMEDOUT; + } + if ((res->cmd & ~0x8000) == creq.cmd) + return -ETIMEDOUT; + /* reply for the previous command most likely arrived */ + } + + if ((res->cmd & ~0x8000) != req->cmd) { + pr_err("bce-vhci: Possible desync, cmd reply mismatch req=%x, res=%x\n", req->cmd, res->cmd); + return -EIO; + } + if (res->status == BCE_VHCI_SUCCESS) + return 0; + return res->status; +} + +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + mutex_lock(&cq->mutex); + status = __bce_vhci_command_queue_execute(cq, req, res, timeout); + mutex_unlock(&cq->mutex); + return status; +} diff --git a/drivers/staging/apple-bce/vhci/queue.h b/drivers/staging/apple-bce/vhci/queue.h new file mode 100644 index 000000000000..adb705b6ba1d --- /dev/null +++ b/drivers/staging/apple-bce/vhci/queue.h @@ -0,0 +1,76 @@ +#ifndef BCE_VHCI_QUEUE_H +#define BCE_VHCI_QUEUE_H + +#include +#include "../queue.h" + +#define VHCI_EVENT_QUEUE_EL_COUNT 256 +#define VHCI_EVENT_PENDING_COUNT 32 + +struct bce_vhci; +struct bce_vhci_event_queue; + +enum bce_vhci_message_status { + BCE_VHCI_SUCCESS = 1, + BCE_VHCI_ERROR = 2, + BCE_VHCI_USB_PIPE_STALL = 3, + BCE_VHCI_ABORT = 4, + BCE_VHCI_BAD_ARGUMENT = 5, + BCE_VHCI_OVERRUN = 6, + BCE_VHCI_INTERNAL_ERROR = 7, + BCE_VHCI_NO_POWER = 8, + BCE_VHCI_UNSUPPORTED = 9 +}; +struct bce_vhci_message { + u16 cmd; + u16 status; // bce_vhci_message_status + u32 param1; + u64 param2; +}; + +struct bce_vhci_message_queue { + struct bce_queue_cq *cq; + struct bce_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; +}; +typedef void (*bce_vhci_event_queue_callback)(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +struct bce_vhci_event_queue { + struct bce_vhci *vhci; + struct bce_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; + bce_vhci_event_queue_callback cb; + struct completion queue_empty_completion; +}; +struct bce_vhci_command_queue_completion { + struct bce_vhci_message *result; + struct completion completion; +}; +struct bce_vhci_command_queue { + struct bce_vhci_message_queue *mq; + struct bce_vhci_command_queue_completion completion; + struct spinlock completion_lock; + struct mutex mutex; +}; + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name); +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q); +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_sq_completion compl); +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb); +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count); +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q); + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq); +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq); +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout); +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg); + +#endif //BCE_VHCI_QUEUE_H diff --git a/drivers/staging/apple-bce/vhci/transfer.c b/drivers/staging/apple-bce/vhci/transfer.c new file mode 100644 index 000000000000..8226363d69c8 --- /dev/null +++ b/drivers/staging/apple-bce/vhci/transfer.c @@ -0,0 +1,661 @@ +#include "transfer.h" +#include "../queue.h" +#include "vhci.h" +#include "../apple_bce.h" +#include + +static void bce_vhci_transfer_queue_completion(struct bce_queue_sq *sq); +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q); +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q); + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb); +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg); +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c); + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work); + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir) +{ + char name[0x21]; + INIT_LIST_HEAD(&q->evq); + INIT_LIST_HEAD(&q->giveback_urb_list); + spin_lock_init(&q->urb_lock); + mutex_init(&q->pause_lock); + q->vhci = vhci; + q->endp = endp; + q->dev_addr = dev_addr; + q->endp_addr = (u8) (endp->desc.bEndpointAddress & 0x8F); + q->state = BCE_VHCI_ENDPOINT_ACTIVE; + q->active = true; + q->stalled = false; + q->max_active_requests = 1; + if (usb_endpoint_type(&endp->desc) == USB_ENDPOINT_XFER_BULK) + q->max_active_requests = BCE_VHCI_BULK_MAX_ACTIVE_URBS; + q->remaining_active_requests = q->max_active_requests; + q->cq = bce_create_cq(vhci->dev, 0x100); + INIT_WORK(&q->w_reset, bce_vhci_transfer_queue_reset_w); + q->sq_in = NULL; + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, 0x80 | usb_endpoint_num(&endp->desc)); + q->sq_in = bce_create_sq(vhci->dev, q->cq, name, 0x100, DMA_FROM_DEVICE, + bce_vhci_transfer_queue_completion, q); + } + q->sq_out = NULL; + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, usb_endpoint_num(&endp->desc)); + q->sq_out = bce_create_sq(vhci->dev, q->cq, name, 0x100, DMA_TO_DEVICE, + bce_vhci_transfer_queue_completion, q); + } +} + +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q) +{ + bce_vhci_transfer_queue_giveback(q); + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + bce_destroy_sq(vhci->dev, q->sq_in); + if (q->sq_out) + bce_destroy_sq(vhci->dev, q->sq_out); + bce_destroy_cq(vhci->dev, q->cq); +} + +static inline bool bce_vhci_transfer_queue_can_init_urb(struct bce_vhci_transfer_queue *q) +{ + return q->remaining_active_requests > 0; +} + +static void bce_vhci_transfer_queue_defer_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + struct bce_vhci_list_message *lm; + lm = kmalloc(sizeof(struct bce_vhci_list_message), GFP_KERNEL); + INIT_LIST_HEAD(&lm->list); + lm->msg = *msg; + list_add_tail(&lm->list, &q->evq); +} + +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + while (!list_empty(&q->giveback_urb_list)) { + urb = list_first_entry(&q->giveback_urb_list, struct urb, urb_list); + list_del(&urb->urb_list); + + spin_unlock_irqrestore(&q->urb_lock, flags); + usb_hcd_giveback_urb(q->vhci->hcd, urb, urb->status); + spin_lock_irqsave(&q->urb_lock, flags); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q); + +static void bce_vhci_transfer_queue_deliver_pending(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb; + struct bce_vhci_list_message *lm; + + while (!list_empty(&q->endp->urb_list) && !list_empty(&q->evq)) { + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + if (bce_vhci_urb_update(urb->hcpriv, &lm->msg) == -EAGAIN) + break; + list_del(&lm->list); + kfree(lm); + } + + /* some of the URBs could have been completed, so initialize more URBs if possible */ + bce_vhci_transfer_queue_init_pending_urbs(q); +} + +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct bce_vhci_list_message *lm; + spin_lock_irqsave(&q->urb_lock, flags); + while (!list_empty(&q->evq)) { + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + list_del(&lm->list); + kfree(lm); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + unsigned long flags; + struct bce_vhci_urb *turb; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + bce_vhci_transfer_queue_deliver_pending(q); + + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && + (!list_empty(&q->evq) || list_empty(&q->endp->urb_list))) { + bce_vhci_transfer_queue_defer_event(q, msg); + goto complete; + } + if (list_empty(&q->endp->urb_list)) { + pr_err("bce-vhci: [%02x] Unexpected transfer queue event\n", q->endp_addr); + goto complete; + } + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + turb = urb->hcpriv; + if (bce_vhci_urb_update(turb, msg) == -EAGAIN) { + bce_vhci_transfer_queue_defer_event(q, msg); + } else { + bce_vhci_transfer_queue_init_pending_urbs(q); + } + +complete: + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +static void bce_vhci_transfer_queue_completion(struct bce_queue_sq *sq) +{ + unsigned long flags; + struct bce_sq_completion_data *c; + struct urb *urb; + struct bce_vhci_transfer_queue *q = sq->userdata; + spin_lock_irqsave(&q->urb_lock, flags); + while ((c = bce_next_completion(sq))) { + if (c->status == BCE_COMPLETION_ABORTED) { /* We flushed the queue */ + pr_debug("bce-vhci: [%02x] Got an abort completion\n", q->endp_addr); + bce_notify_submission_complete(sq); + continue; + } + if (list_empty(&q->endp->urb_list)) { + pr_err("bce-vhci: [%02x] Got a completion while no requests are pending\n", q->endp_addr); + continue; + } + pr_debug("bce-vhci: [%02x] Got a transfer queue completion\n", q->endp_addr); + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + bce_vhci_urb_transfer_completion(urb->hcpriv, c); + bce_notify_submission_complete(sq); + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + spin_lock_irqsave(&q->urb_lock, flags); + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + if (q->sq_out) { + pr_err("bce-vhci: Not implemented: wait for pending output requests\n"); + } + bce_vhci_transfer_queue_remove_pending(q); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) + return status; + if (q->state != BCE_VHCI_ENDPOINT_PAUSED) + return -EINVAL; + if (q->sq_in) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_in->qid); + if (q->sq_out) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_out->qid); + return 0; +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb); + +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_ACTIVE, &q->state))) + return status; + if (q->state != BCE_VHCI_ENDPOINT_ACTIVE) + return -EINVAL; + spin_lock_irqsave(&q->urb_lock, flags); + q->active = true; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) { + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + bce_vhci_urb_init(vurb); + } else { + bce_vhci_urb_resume(vurb); + } + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + return 0; +} + +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + mutex_lock(&q->pause_lock); + if ((q->paused_by & src) != src) { + if (!q->paused_by) + ret = bce_vhci_transfer_queue_do_pause(q); + if (!ret) + q->paused_by |= src; + } + mutex_unlock(&q->pause_lock); + return ret; +} + +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + mutex_lock(&q->pause_lock); + if (q->paused_by & src) { + if (!(q->paused_by & ~src)) + ret = bce_vhci_transfer_queue_do_resume(q); + if (!ret) + q->paused_by &= ~src; + } + mutex_unlock(&q->pause_lock); + return ret; +} + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work) +{ + unsigned long flags; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_reset); + + mutex_lock(&q->pause_lock); + spin_lock_irqsave(&q->urb_lock, flags); + if (!q->stalled) { + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + return; + } + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + q->paused_by |= BCE_VHCI_PAUSE_INTERNAL_WQ; + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_in->qid); + if (q->sq_out) + bce_cmd_flush_memory_queue(q->vhci->dev->cmd_cmdq, (u16) q->sq_out->qid); + bce_vhci_cmd_endpoint_reset(&q->vhci->cq, q->dev_addr, (u8) (q->endp->desc.bEndpointAddress & 0x8F)); + spin_lock_irqsave(&q->urb_lock, flags); + q->stalled = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); +} + +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q) +{ + queue_work(q->vhci->tq_state_wq, &q->w_reset); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) + bce_vhci_urb_init(vurb); + } +} + + + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout); + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb) +{ + unsigned long flags; + int status = 0; + struct bce_vhci_urb *vurb; + vurb = kzalloc(sizeof(struct bce_vhci_urb), GFP_KERNEL); + urb->hcpriv = vurb; + + vurb->q = q; + vurb->urb = urb; + vurb->dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + vurb->is_control = (usb_endpoint_num(&urb->ep->desc) == 0); + + spin_lock_irqsave(&q->urb_lock, flags); + status = usb_hcd_link_urb_to_ep(q->vhci->hcd, urb); + if (status) { + spin_unlock_irqrestore(&q->urb_lock, flags); + urb->hcpriv = NULL; + kfree(vurb); + return status; + } + + if (q->active) { + if (bce_vhci_transfer_queue_can_init_urb(vurb->q)) + status = bce_vhci_urb_init(vurb); + else + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } else { + if (q->stalled) + bce_vhci_transfer_queue_request_reset(q); + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } + if (status) { + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + urb->hcpriv = NULL; + kfree(vurb); + } else { + bce_vhci_transfer_queue_deliver_pending(q); + } + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("bce-vhci: [%02x] URB enqueued (dir = %s, size = %i)\n", q->endp_addr, + usb_urb_dir_in(urb) ? "IN" : "OUT", urb->transfer_buffer_length); + return status; +} + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb) +{ + int status = 0; + + if (vurb->q->remaining_active_requests == 0) { + pr_err("bce-vhci: cannot init request (remaining_active_requests = 0)\n"); + return -EINVAL; + } + + if (vurb->is_control) { + vurb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST; + } else { + status = bce_vhci_urb_data_start(vurb, NULL); + } + + if (!status) { + --vurb->q->remaining_active_requests; + } + return status; +} + +static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->q; + struct bce_vhci *vhci = q->vhci; + struct urb *real_urb = urb->urb; + pr_debug("bce-vhci: [%02x] URB complete %i\n", q->endp_addr, status); + usb_hcd_unlink_urb_from_ep(vhci->hcd, real_urb); + real_urb->hcpriv = NULL; + real_urb->status = status; + if (urb->state != BCE_VHCI_URB_INIT_PENDING) + ++urb->q->remaining_active_requests; + kfree(urb); + list_add_tail(&real_urb->urb_list, &q->giveback_urb_list); +} + +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status) +{ + struct bce_vhci_urb *vurb; + unsigned long flags; + int ret; + + spin_lock_irqsave(&q->urb_lock, flags); + if ((ret = usb_hcd_check_unlink_urb(q->vhci->hcd, urb, status))) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return ret; + } + + vurb = urb->hcpriv; + /* If the URB wasn't posted to the device yet, we can still remove it on the host without pausing the queue. */ + if (vurb->state != BCE_VHCI_URB_INIT_PENDING) { + pr_debug("bce-vhci: [%02x] Cancelling URB\n", q->endp_addr); + + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_pause(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + spin_lock_irqsave(&q->urb_lock, flags); + + ++q->remaining_active_requests; + } + + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + + spin_unlock_irqrestore(&q->urb_lock, flags); + + usb_hcd_giveback_urb(q->vhci->hcd, urb, status); + + if (vurb->state != BCE_VHCI_URB_INIT_PENDING) + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + + kfree(vurb); + + return 0; +} + +static int bce_vhci_urb_data_transfer_in(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + struct bce_vhci_message msg; + struct bce_qe_submission *s; + u32 tr_len; + int reservation1, reservation2 = -EFAULT; + + pr_debug("bce-vhci: [%02x] DMA from device %llx %x\n", urb->q->endp_addr, + (u64) urb->urb->transfer_dma, urb->urb->transfer_buffer_length); + + /* Reserve both a message and a submission, so we don't run into issues later. */ + reservation1 = bce_reserve_submission(urb->q->vhci->msg_asynchronous.sq, timeout); + if (!reservation1) + reservation2 = bce_reserve_submission(urb->q->sq_in, timeout); + if (reservation1 || reservation2) { + pr_err("bce-vhci: Failed to reserve a submission for URB data transfer\n"); + if (!reservation1) + bce_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); + return -ENOMEM; + } + + urb->send_offset = urb->receive_offset; + + tr_len = urb->urb->transfer_buffer_length - urb->send_offset; + + spin_lock(&urb->q->vhci->msg_asynchronous_lock); + msg.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + msg.status = 0; + msg.param1 = ((urb->urb->ep->desc.bEndpointAddress & 0x8Fu) << 8) | urb->q->dev_addr; + msg.param2 = tr_len; + bce_vhci_message_queue_write(&urb->q->vhci->msg_asynchronous, &msg); + spin_unlock(&urb->q->vhci->msg_asynchronous_lock); + + s = bce_next_submission(urb->q->sq_in); + bce_set_submission_single(s, urb->urb->transfer_dma + urb->send_offset, tr_len); + bce_submit_to_device(urb->q->sq_in); + + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + return 0; +} + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + if (urb->dir == DMA_TO_DEVICE) { + if (urb->urb->transfer_buffer_length > 0) + urb->state = BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST; + else + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + return 0; + } else { + return bce_vhci_urb_data_transfer_in(urb, timeout); + } +} + +static int bce_vhci_urb_send_out_data(struct bce_vhci_urb *urb, dma_addr_t addr, size_t size) +{ + struct bce_qe_submission *s; + unsigned long timeout = 0; + if (bce_reserve_submission(urb->q->sq_out, &timeout)) { + pr_err("bce-vhci: Failed to reserve a submission for URB data transfer\n"); + return -EPIPE; + } + + pr_debug("bce-vhci: [%02x] DMA to device %llx %lx\n", urb->q->endp_addr, (u64) addr, size); + + s = bce_next_submission(urb->q->sq_out); + bce_set_submission_single(s, addr, size); + bce_submit_to_device(urb->q->sq_out); + return 0; +} + +static int bce_vhci_urb_data_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + u32 tr_len; + int status; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + tr_len = min(urb->urb->transfer_buffer_length - urb->send_offset, (u32) msg->param2); + if ((status = bce_vhci_urb_send_out_data(urb, urb->urb->transfer_dma + urb->send_offset, tr_len))) + return status; + urb->send_offset += tr_len; + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + return 0; + } + } + + /* 0x1000 in out queues aren't really unexpected */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("bce-vhci: [%02x] %s URB unexpected message (state = %x, msg: %x %x %x %llx)\n", + urb->q->endp_addr, (urb->is_control ? "Control (data update)" : "Data"), urb->state, + msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_data_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) +{ + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + urb->receive_offset += c->data_size; + if (urb->dir == DMA_FROM_DEVICE || urb->receive_offset >= urb->urb->transfer_buffer_length) { + urb->urb->actual_length = (u32) urb->receive_offset; + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + if (!urb->is_control) { + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + } + } else { + pr_err("bce-vhci: [%02x] Data URB unexpected completion\n", urb->q->endp_addr); + } + return 0; +} + + +static int bce_vhci_urb_control_check_status(struct bce_vhci_urb *urb) +{ + struct bce_vhci_transfer_queue *q = urb->q; + if (urb->received_status == 0) + return 0; + if (urb->state == BCE_VHCI_URB_DATA_TRANSFER_COMPLETE || + (urb->received_status != BCE_VHCI_SUCCESS && urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST && + urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION)) { + urb->state = BCE_VHCI_URB_CONTROL_COMPLETE; + if (urb->received_status != BCE_VHCI_SUCCESS) { + pr_err("bce-vhci: [%02x] URB failed: %x\n", urb->q->endp_addr, urb->received_status); + urb->q->active = false; + urb->q->stalled = true; + bce_vhci_urb_complete(urb, -EPIPE); + if (!list_empty(&q->endp->urb_list)) + bce_vhci_transfer_queue_request_reset(q); + return -ENOENT; + } + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + return 0; +} + +static int bce_vhci_urb_control_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + int status; + if (msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + urb->received_status = msg->status; + return bce_vhci_urb_control_check_status(urb); + } + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + if (bce_vhci_urb_send_out_data(urb, urb->urb->setup_dma, sizeof(struct usb_ctrlrequest))) { + pr_err("bce-vhci: [%02x] Failed to start URB setup transfer\n", urb->q->endp_addr); + return 0; /* TODO: fail the URB? */ + } + urb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION; + pr_debug("bce-vhci: [%02x] Sent setup %llx\n", urb->q->endp_addr, urb->urb->setup_dma); + return 0; + } + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_update(urb, msg))) + return status; + return bce_vhci_urb_control_check_status(urb); + } + + /* 0x1000 in out queues aren't really unexpected */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("bce-vhci: [%02x] Control URB unexpected message (state = %x, msg: %x %x %x %llx)\n", urb->q->endp_addr, + urb->state, msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_control_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) +{ + int status; + unsigned long timeout; + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION) { + if (c->data_size != sizeof(struct usb_ctrlrequest)) + pr_err("bce-vhci: [%02x] transfer complete data size mistmatch for usb_ctrlrequest (%llx instead of %lx)\n", + urb->q->endp_addr, c->data_size, sizeof(struct usb_ctrlrequest)); + + timeout = 1000; + status = bce_vhci_urb_data_start(urb, &timeout); + if (status) { + bce_vhci_urb_complete(urb, status); + return -ENOENT; + } + return 0; + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_transfer_completion(urb, c))) + return status; + return bce_vhci_urb_control_check_status(urb); + } else { + pr_err("bce-vhci: [%02x] Control URB unexpected completion (state = %x)\n", urb->q->endp_addr, urb->state); + } + return 0; +} + +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + if (urb->state == BCE_VHCI_URB_INIT_PENDING) + return -EAGAIN; + if (urb->is_control) + return bce_vhci_urb_control_update(urb, msg); + else + return bce_vhci_urb_data_update(urb, msg); +} + +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct bce_sq_completion_data *c) +{ + if (urb->is_control) + return bce_vhci_urb_control_transfer_completion(urb, c); + else + return bce_vhci_urb_data_transfer_completion(urb, c); +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb) +{ + int status = 0; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + status = bce_vhci_urb_data_transfer_in(urb, NULL); + } + if (status) + bce_vhci_urb_complete(urb, status); +} diff --git a/drivers/staging/apple-bce/vhci/transfer.h b/drivers/staging/apple-bce/vhci/transfer.h new file mode 100644 index 000000000000..89ecad6bcf8f --- /dev/null +++ b/drivers/staging/apple-bce/vhci/transfer.h @@ -0,0 +1,73 @@ +#ifndef BCEDRIVER_TRANSFER_H +#define BCEDRIVER_TRANSFER_H + +#include +#include "queue.h" +#include "command.h" +#include "../queue.h" + +struct bce_vhci_list_message { + struct list_head list; + struct bce_vhci_message msg; +}; +enum bce_vhci_pause_source { + BCE_VHCI_PAUSE_INTERNAL_WQ = 1, + BCE_VHCI_PAUSE_FIRMWARE = 2, + BCE_VHCI_PAUSE_SUSPEND = 4, + BCE_VHCI_PAUSE_SHUTDOWN = 8 +}; +struct bce_vhci_transfer_queue { + struct bce_vhci *vhci; + struct usb_host_endpoint *endp; + enum bce_vhci_endpoint_state state; + u32 max_active_requests, remaining_active_requests; + bool active, stalled; + u32 paused_by; + bce_vhci_device_t dev_addr; + u8 endp_addr; + struct bce_queue_cq *cq; + struct bce_queue_sq *sq_in; + struct bce_queue_sq *sq_out; + struct list_head evq; + struct spinlock urb_lock; + struct mutex pause_lock; + struct list_head giveback_urb_list; + + struct work_struct w_reset; +}; +enum bce_vhci_urb_state { + BCE_VHCI_URB_INIT_PENDING, + + BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST, + BCE_VHCI_URB_WAITING_FOR_COMPLETION, + BCE_VHCI_URB_DATA_TRANSFER_COMPLETE, + + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST, + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION, + BCE_VHCI_URB_CONTROL_COMPLETE +}; +struct bce_vhci_urb { + struct urb *urb; + struct bce_vhci_transfer_queue *q; + enum dma_data_direction dir; + bool is_control; + enum bce_vhci_urb_state state; + int received_status; + u32 send_offset; + u32 receive_offset; +}; + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir); +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q); +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg); +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q); + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb); +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status); + +#endif //BCEDRIVER_TRANSFER_H diff --git a/drivers/staging/apple-bce/vhci/vhci.c b/drivers/staging/apple-bce/vhci/vhci.c new file mode 100644 index 000000000000..eb26f55000d8 --- /dev/null +++ b/drivers/staging/apple-bce/vhci/vhci.c @@ -0,0 +1,759 @@ +#include "vhci.h" +#include "../apple_bce.h" +#include "command.h" +#include +#include +#include +#include + +static dev_t bce_vhci_chrdev; +static struct class *bce_vhci_class; +static const struct hc_driver bce_vhci_driver; +static u16 bce_vhci_port_mask = U16_MAX; + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci); +static int bce_vhci_create_message_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci); +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws); +static void bce_vhci_firmware_event_completion(struct bce_queue_sq *sq); + +int bce_vhci_create(struct apple_bce_device *dev, struct bce_vhci *vhci) +{ + int status; + + spin_lock_init(&vhci->hcd_spinlock); + + vhci->dev = dev; + + vhci->vdevt = bce_vhci_chrdev; + vhci->vdev = device_create(bce_vhci_class, dev->dev, vhci->vdevt, NULL, "bce-vhci"); + if (IS_ERR_OR_NULL(vhci->vdev)) { + status = PTR_ERR(vhci->vdev); + goto fail_dev; + } + + if ((status = bce_vhci_create_message_queues(vhci))) + goto fail_mq; + if ((status = bce_vhci_create_event_queues(vhci))) + goto fail_eq; + + vhci->tq_state_wq = alloc_ordered_workqueue("bce-vhci-tq-state", 0); + INIT_WORK(&vhci->w_fw_events, bce_vhci_handle_firmware_events_w); + + vhci->hcd = usb_create_hcd(&bce_vhci_driver, vhci->vdev, "bce-vhci"); + if (!vhci->hcd) { + status = -ENOMEM; + goto fail_hcd; + } + vhci->hcd->self.sysdev = &dev->pci->dev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + vhci->hcd->self.uses_dma = 1; +#endif + *((struct bce_vhci **) vhci->hcd->hcd_priv) = vhci; + vhci->hcd->speed = HCD_USB2; + + if ((status = usb_add_hcd(vhci->hcd, 0, 0))) + goto fail_hcd; + + return 0; + +fail_hcd: + bce_vhci_destroy_event_queues(vhci); +fail_eq: + bce_vhci_destroy_message_queues(vhci); +fail_mq: + device_destroy(bce_vhci_class, vhci->vdevt); +fail_dev: + if (!status) + status = -EINVAL; + return status; +} + +void bce_vhci_destroy(struct bce_vhci *vhci) +{ + usb_remove_hcd(vhci->hcd); + bce_vhci_destroy_event_queues(vhci); + bce_vhci_destroy_message_queues(vhci); + device_destroy(bce_vhci_class, vhci->vdevt); +} + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd) +{ + return *((struct bce_vhci **) hcd->hcd_priv); +} + +int bce_vhci_start(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + u16 port_mask = 0; + bce_vhci_port_t port_no = 0; + if ((status = bce_vhci_cmd_controller_enable(&vhci->cq, 1, &port_mask))) + return status; + vhci->port_mask = port_mask; + vhci->port_power_mask = 0; + if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) + return status; + port_mask = vhci->port_mask; + while (port_mask) { + port_no += 1; + port_mask >>= 1; + } + vhci->port_count = port_no; + return 0; +} + +void bce_vhci_stop(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_cmd_controller_disable(&vhci->cq); +} + +static int bce_vhci_hub_status_data(struct usb_hcd *hcd, char *buf) +{ + return 0; +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout); + +static int bce_vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + struct usb_hub_descriptor *hd; + struct usb_hub_status *hs; + struct usb_port_status *ps; + u32 port_status; + // pr_info("bce-vhci: bce_vhci_hub_control %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); + if (typeReq == GetHubDescriptor && wLength >= sizeof(struct usb_hub_descriptor)) { + hd = (struct usb_hub_descriptor *) buf; + memset(hd, 0, sizeof(*hd)); + hd->bDescLength = sizeof(struct usb_hub_descriptor); + hd->bDescriptorType = USB_DT_HUB; + hd->bNbrPorts = (u8) vhci->port_count; + hd->wHubCharacteristics = HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_INDV_PORT_OCPM; + hd->bPwrOn2PwrGood = 0; + hd->bHubContrCurrent = 0; + return 0; + } else if (typeReq == GetHubStatus && wLength >= sizeof(struct usb_hub_status)) { + hs = (struct usb_hub_status *) buf; + memset(hs, 0, sizeof(*hs)); + hs->wHubStatus = 0; + hs->wHubChange = 0; + return 0; + } else if (typeReq == GetPortStatus && wLength >= 4 /* usb 2.0 */) { + ps = (struct usb_port_status *) buf; + ps->wPortStatus = 0; + ps->wPortChange = 0; + + if (vhci->port_power_mask & BIT(wIndex)) + ps->wPortStatus |= USB_PORT_STAT_POWER; + + if (!(bce_vhci_port_mask & BIT(wIndex))) + return 0; + + if ((status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0, &port_status))) + return status; + + if (port_status & 16) + ps->wPortStatus |= USB_PORT_STAT_ENABLE | USB_PORT_STAT_HIGH_SPEED; + if (port_status & 4) + ps->wPortStatus |= USB_PORT_STAT_CONNECTION; + if (port_status & 2) + ps->wPortStatus |= USB_PORT_STAT_OVERCURRENT; + if (port_status & 8) + ps->wPortStatus |= USB_PORT_STAT_RESET; + if (port_status & 0x60) + ps->wPortStatus |= USB_PORT_STAT_SUSPEND; + + if (port_status & 0x40000) + ps->wPortChange |= USB_PORT_STAT_C_CONNECTION; + + pr_debug("bce-vhci: Translated status %x to %x:%x\n", port_status, ps->wPortStatus, ps->wPortChange); + return 0; + } else if (typeReq == SetPortFeature) { + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) wIndex); + /* As far as I am aware, power status is not part of the port status so store it separately */ + if (!status) + vhci->port_power_mask |= BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_RESET) { + return bce_vhci_reset_device(vhci, wIndex, wValue); + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + /* TODO: Am I supposed to also suspend the endpoints? */ + pr_debug("bce-vhci: Suspending port %i\n", wIndex); + return bce_vhci_cmd_port_suspend(&vhci->cq, (u8) wIndex); + } + } else if (typeReq == ClearPortFeature) { + if (wValue == USB_PORT_FEAT_ENABLE) + return bce_vhci_cmd_port_disable(&vhci->cq, (u8) wIndex); + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) wIndex); + if (!status) + vhci->port_power_mask &= ~BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_C_CONNECTION) + return bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &port_status); + if (wValue == USB_PORT_FEAT_C_RESET) { /* I don't think I can transfer it in any way */ + return 0; + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + pr_debug("bce-vhci: Resuming port %i\n", wIndex); + return bce_vhci_cmd_port_resume(&vhci->cq, (u8) wIndex); + } + } + pr_err("bce-vhci: bce_vhci_hub_control unhandled request: %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); + dump_stack(); + return -EIO; +} + +static int bce_vhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + struct bce_vhci_device *vdev; + bce_vhci_device_t devid; + pr_info("bce_vhci_enable_device\n"); + + if (vhci->port_to_device[udev->portnum]) + return 0; + + /* We need to early address the device */ + if (bce_vhci_cmd_device_create(&vhci->cq, udev->portnum, &devid)) + return -EIO; + + pr_info("bce_vhci_cmd_device_create %i -> %i\n", udev->portnum, devid); + + vdev = kzalloc(sizeof(struct bce_vhci_device), GFP_KERNEL); + vhci->port_to_device[udev->portnum] = devid; + vhci->devices[devid] = vdev; + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[0], &udev->ep0, devid, DMA_BIDIRECTIONAL); + udev->ep0.hcpriv = &vdev->tq[0]; + vdev->tq_mask |= BIT(0); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &udev->ep0.desc); + return 0; +} + +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, unsigned int timeout_ms) //TODO: follow timeout +{ + /* This is the same as enable_device, but instead in the old scheme */ + return bce_vhci_enable_device(hcd, udev); +} + +static void bce_vhci_free_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int i; + bce_vhci_device_t devid; + struct bce_vhci_device *dev; + pr_info("bce_vhci_free_device %i\n", udev->portnum); + if (!vhci->port_to_device[udev->portnum]) + return; + devid = vhci->port_to_device[udev->portnum]; + dev = vhci->devices[devid]; + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) i); + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[udev->portnum] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + kfree(dev); +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout) +{ + struct bce_vhci_device *dev = NULL; + bce_vhci_device_t devid; + int i; + int status; + enum dma_data_direction dir; + pr_info("bce_vhci_reset_device %i\n", index); + + devid = vhci->port_to_device[index]; + if (devid) { + dev = vhci->devices[devid]; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) i); + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[index] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + } + status = bce_vhci_cmd_port_reset(&vhci->cq, (u8) index, timeout); + + if (dev) { + if ((status = bce_vhci_cmd_device_create(&vhci->cq, index, &devid))) + return status; + vhci->devices[devid] = dev; + vhci->port_to_device[index] = devid; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + dir = usb_endpoint_dir_in(&dev->tq[i].endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + if (i == 0) + dir = DMA_BIDIRECTIONAL; + bce_vhci_create_transfer_queue(vhci, &dev->tq[i], dev->tq[i].endp, devid, dir); + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &dev->tq[i].endp->desc); + } + } + } + + return status; +} + +static int bce_vhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) +{ + return 0; +} + +static int bce_vhci_get_frame_number(struct usb_hcd *hcd) +{ + return 0; +} + +static int bce_vhci_bus_suspend(struct usb_hcd *hcd) +{ + int i, j; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + pr_info("bce_vhci: suspend started\n"); + + pr_info("bce_vhci: suspend endpoints\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + for (j = 0; j < 32; j++) { + if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) + continue; + bce_vhci_transfer_queue_pause(&vhci->devices[vhci->port_to_device[i]]->tq[j], + BCE_VHCI_PAUSE_SUSPEND); + } + } + + pr_info("bce_vhci: suspend ports\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + bce_vhci_cmd_port_suspend(&vhci->cq, i); + } + pr_info("bce_vhci: suspend controller\n"); + if ((status = bce_vhci_cmd_controller_pause(&vhci->cq))) + return status; + + bce_vhci_event_queue_pause(&vhci->ev_commands); + bce_vhci_event_queue_pause(&vhci->ev_system); + bce_vhci_event_queue_pause(&vhci->ev_isochronous); + bce_vhci_event_queue_pause(&vhci->ev_interrupt); + bce_vhci_event_queue_pause(&vhci->ev_asynchronous); + pr_info("bce_vhci: suspend done\n"); + return 0; +} + +static int bce_vhci_bus_resume(struct usb_hcd *hcd) +{ + int i, j; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + pr_info("bce_vhci: resume started\n"); + + bce_vhci_event_queue_resume(&vhci->ev_system); + bce_vhci_event_queue_resume(&vhci->ev_isochronous); + bce_vhci_event_queue_resume(&vhci->ev_interrupt); + bce_vhci_event_queue_resume(&vhci->ev_asynchronous); + bce_vhci_event_queue_resume(&vhci->ev_commands); + + pr_info("bce_vhci: resume controller\n"); + if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) + return status; + + pr_info("bce_vhci: resume ports\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + bce_vhci_cmd_port_resume(&vhci->cq, i); + } + pr_info("bce_vhci: resume endpoints\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + for (j = 0; j < 32; j++) { + if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) + continue; + bce_vhci_transfer_queue_resume(&vhci->devices[vhci->port_to_device[i]]->tq[j], + BCE_VHCI_PAUSE_SUSPEND); + } + } + + pr_info("bce_vhci: resume done\n"); + return 0; +} + +static int bce_vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + pr_debug("bce_vhci_urb_enqueue %i:%x\n", q->dev_addr, urb->ep->desc.bEndpointAddress); + if (!q) + return -ENOENT; + return bce_vhci_urb_create(q, urb); +} + +static int bce_vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + pr_debug("bce_vhci_urb_dequeue %x\n", urb->ep->desc.bEndpointAddress); + return bce_vhci_urb_request_cancel(q, urb, status); +} + +static void bce_vhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep) +{ + struct bce_vhci_transfer_queue *q = ep->hcpriv; + pr_debug("bce_vhci_endpoint_reset\n"); + if (q) + bce_vhci_transfer_queue_request_reset(q); +} + +static u8 bce_vhci_endpoint_index(u8 addr) +{ + if (addr & 0x80) + return (u8) (0x10 + (addr & 0xf)); + return (u8) (addr & 0xf); +} + +static int bce_vhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_device *vdev = vhci->devices[devid]; + pr_debug("bce_vhci_add_endpoint %x/%x:%x\n", udev->portnum, devid, endp_index); + + if (udev->bus->root_hub == udev) /* The USB hub */ + return 0; + if (vdev == NULL) + return -ENODEV; + if (vdev->tq_mask & BIT(endp_index)) { + endp->hcpriv = &vdev->tq[endp_index]; + return 0; + } + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[endp_index], endp, devid, + usb_endpoint_dir_in(&endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); + endp->hcpriv = &vdev->tq[endp_index]; + vdev->tq_mask |= BIT(endp_index); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &endp->desc); + return 0; +} + +static int bce_vhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_transfer_queue *q = endp->hcpriv; + struct bce_vhci_device *vdev = vhci->devices[devid]; + pr_info("bce_vhci_drop_endpoint %x:%x\n", udev->portnum, endp_index); + if (!q) { + if (vdev && vdev->tq_mask & BIT(endp_index)) { + pr_err("something deleted the hcpriv?\n"); + q = &vdev->tq[endp_index]; + } else { + return 0; + } + } + + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) (endp->desc.bEndpointAddress & 0x8Fu)); + vhci->devices[devid]->tq_mask &= ~BIT(endp_index); + bce_vhci_destroy_transfer_queue(vhci, q); + return 0; +} + +static int bce_vhci_create_message_queues(struct bce_vhci *vhci) +{ + if (bce_vhci_message_queue_create(vhci, &vhci->msg_commands, "VHC1HostCommands") || + bce_vhci_message_queue_create(vhci, &vhci->msg_system, "VHC1HostSystemEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_isochronous, "VHC1HostIsochronousEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_interrupt, "VHC1HostInterruptEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_asynchronous, "VHC1HostAsynchronousEvents")) { + bce_vhci_destroy_message_queues(vhci); + return -EINVAL; + } + spin_lock_init(&vhci->msg_asynchronous_lock); + bce_vhci_command_queue_create(&vhci->cq, &vhci->msg_commands); + return 0; +} + +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci) +{ + bce_vhci_command_queue_destroy(&vhci->cq); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_commands); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_system); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_isochronous); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_interrupt); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_asynchronous); +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci) +{ + vhci->ev_cq = bce_create_cq(vhci->dev, 0x100); + if (!vhci->ev_cq) + return -EINVAL; +#define CREATE_EVENT_QUEUE(field, name, cb) bce_vhci_event_queue_create(vhci, &vhci->field, name, cb) + if (__bce_vhci_event_queue_create(vhci, &vhci->ev_commands, "VHC1FirmwareCommands", + bce_vhci_firmware_event_completion) || + CREATE_EVENT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", bce_vhci_handle_system_event) || + CREATE_EVENT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", bce_vhci_handle_usb_event)) { + bce_vhci_destroy_event_queues(vhci); + return -EINVAL; + } +#undef CREATE_EVENT_QUEUE + return 0; +} + +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci) +{ + bce_vhci_event_queue_destroy(vhci, &vhci->ev_commands); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_system); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_isochronous); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_interrupt); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_asynchronous); + if (vhci->ev_cq) + bce_destroy_cq(vhci->dev, vhci->ev_cq); +} + +static void bce_vhci_send_fw_event_response(struct bce_vhci *vhci, struct bce_vhci_message *req, u16 status) +{ + unsigned long timeout = 1000; + struct bce_vhci_message r = *req; + r.cmd = (u16) (req->cmd | 0x8000u); + r.status = status; + r.param1 = req->param1; + r.param2 = 0; + + if (bce_reserve_submission(vhci->msg_system.sq, &timeout)) { + pr_err("bce-vhci: Cannot reserve submision for FW event reply\n"); + return; + } + bce_vhci_message_queue_write(&vhci->msg_system, &r); +} + +static int bce_vhci_handle_firmware_event(struct bce_vhci *vhci, struct bce_vhci_message *msg) +{ + unsigned long flags; + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + struct bce_vhci_transfer_queue *tq; + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE || msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = vhci->devices[devid]; + if (!dev || !(dev->tq_mask & BIT(endp))) + return BCE_VHCI_BAD_ARGUMENT; + tq = &dev->tq[endp]; + } + + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_ACTIVE) { + bce_vhci_transfer_queue_resume(tq, BCE_VHCI_PAUSE_FIRMWARE); + return BCE_VHCI_SUCCESS; + } else if (msg->param2 == BCE_VHCI_ENDPOINT_PAUSED) { + bce_vhci_transfer_queue_pause(tq, BCE_VHCI_PAUSE_FIRMWARE); + return BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } else if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_STALLED) { + tq->state = msg->param2; + spin_lock_irqsave(&tq->urb_lock, flags); + tq->stalled = true; + spin_unlock_irqrestore(&tq->urb_lock, flags); + return BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } + pr_warn("bce-vhci: Unhandled firmware event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + return BCE_VHCI_BAD_ARGUMENT; +} + +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws) +{ + size_t cnt = 0; + int result; + struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_fw_events); + struct bce_queue_sq *sq = vhci->ev_commands.sq; + struct bce_sq_completion_data *cq; + struct bce_vhci_message *msg, *msg2 = NULL; + + while (true) { + if (msg2) { + msg = msg2; + msg2 = NULL; + } else if ((cq = bce_next_completion(sq))) { + if (cq->status == BCE_COMPLETION_ABORTED) { + bce_notify_submission_complete(sq); + continue; + } + msg = &vhci->ev_commands.data[sq->head]; + } else { + break; + } + + pr_debug("bce-vhci: Got fw event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + if ((cq = bce_next_completion(sq))) { + msg2 = &vhci->ev_commands.data[(sq->head + 1) % sq->el_count]; + pr_debug("bce-vhci: Got second fw event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + if (cq->status != BCE_COMPLETION_ABORTED && + msg2->cmd == (msg->cmd | 0x4000) && msg2->param1 == msg->param1) { + /* Take two elements */ + pr_debug("bce-vhci: Cancelled\n"); + bce_vhci_send_fw_event_response(vhci, msg, BCE_VHCI_ABORT); + + bce_notify_submission_complete(sq); + bce_notify_submission_complete(sq); + msg2 = NULL; + cnt += 2; + continue; + } + + pr_warn("bce-vhci: Handle fw event - unexpected cancellation\n"); + } + + result = bce_vhci_handle_firmware_event(vhci, msg); + bce_vhci_send_fw_event_response(vhci, msg, (u16) result); + + + bce_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(&vhci->ev_commands, cnt); + if (atomic_read(&sq->available_commands) == sq->el_count - 1) { + pr_debug("bce-vhci: complete\n"); + complete(&vhci->ev_commands.queue_empty_completion); + } +} + +static void bce_vhci_firmware_event_completion(struct bce_queue_sq *sq) +{ + struct bce_vhci_event_queue *q = sq->userdata; + queue_work(q->vhci->tq_state_wq, &q->vhci->w_fw_events); +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else { + pr_warn("bce-vhci: Unhandled system event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST || msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = q->vhci->devices[devid]; + if (!dev || (dev->tq_mask & BIT(endp)) == 0) { + pr_err("bce-vhci: Didn't find destination for transfer queue event\n"); + return; + } + bce_vhci_transfer_queue_event(&dev->tq[endp], msg); + } else { + pr_warn("bce-vhci: Unhandled USB event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + + + +static const struct hc_driver bce_vhci_driver = { + .description = "bce-vhci", + .product_desc = "BCE VHCI Host Controller", + .hcd_priv_size = sizeof(struct bce_vhci *), + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + .flags = HCD_USB2, +#else + .flags = HCD_USB2 | HCD_DMA, +#endif + + .start = bce_vhci_start, + .stop = bce_vhci_stop, + .hub_status_data = bce_vhci_hub_status_data, + .hub_control = bce_vhci_hub_control, + .urb_enqueue = bce_vhci_urb_enqueue, + .urb_dequeue = bce_vhci_urb_dequeue, + .enable_device = bce_vhci_enable_device, + .free_dev = bce_vhci_free_device, + .address_device = bce_vhci_address_device, + .add_endpoint = bce_vhci_add_endpoint, + .drop_endpoint = bce_vhci_drop_endpoint, + .endpoint_reset = bce_vhci_endpoint_reset, + .check_bandwidth = bce_vhci_check_bandwidth, + .get_frame_number = bce_vhci_get_frame_number, + .bus_suspend = bce_vhci_bus_suspend, + .bus_resume = bce_vhci_bus_resume +}; + + +int __init bce_vhci_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&bce_vhci_chrdev, 0, 1, "bce-vhci"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_vhci_class = class_create(THIS_MODULE, "bce-vhci"); +#else + bce_vhci_class = class_create("bce-vhci"); +#endif + if (IS_ERR(bce_vhci_class)) { + result = PTR_ERR(bce_vhci_class); + goto fail_class; + } + return 0; + +fail_class: + class_destroy(bce_vhci_class); +fail_chrdev: + unregister_chrdev_region(bce_vhci_chrdev, 1); + if (!result) + result = -EINVAL; + return result; +} +void __exit bce_vhci_module_exit(void) +{ + class_destroy(bce_vhci_class); + unregister_chrdev_region(bce_vhci_chrdev, 1); +} + +module_param_named(vhci_port_mask, bce_vhci_port_mask, ushort, 0444); +MODULE_PARM_DESC(vhci_port_mask, "Specifies which VHCI ports are enabled"); diff --git a/drivers/staging/apple-bce/vhci/vhci.h b/drivers/staging/apple-bce/vhci/vhci.h new file mode 100644 index 000000000000..6c2e22622f4c --- /dev/null +++ b/drivers/staging/apple-bce/vhci/vhci.h @@ -0,0 +1,52 @@ +#ifndef BCE_VHCI_H +#define BCE_VHCI_H + +#include "queue.h" +#include "transfer.h" + +struct usb_hcd; +struct bce_queue_cq; + +struct bce_vhci_device { + struct bce_vhci_transfer_queue tq[32]; + u32 tq_mask; +}; +struct bce_vhci { + struct apple_bce_device *dev; + dev_t vdevt; + struct device *vdev; + struct usb_hcd *hcd; + struct spinlock hcd_spinlock; + struct bce_vhci_message_queue msg_commands; + struct bce_vhci_message_queue msg_system; + struct bce_vhci_message_queue msg_isochronous; + struct bce_vhci_message_queue msg_interrupt; + struct bce_vhci_message_queue msg_asynchronous; + struct spinlock msg_asynchronous_lock; + struct bce_vhci_command_queue cq; + struct bce_queue_cq *ev_cq; + struct bce_vhci_event_queue ev_commands; + struct bce_vhci_event_queue ev_system; + struct bce_vhci_event_queue ev_isochronous; + struct bce_vhci_event_queue ev_interrupt; + struct bce_vhci_event_queue ev_asynchronous; + u16 port_mask; + u8 port_count; + u16 port_power_mask; + bce_vhci_device_t port_to_device[16]; + struct bce_vhci_device *devices[16]; + struct workqueue_struct *tq_state_wq; + struct work_struct w_fw_events; +}; + +int __init bce_vhci_module_init(void); +void __exit bce_vhci_module_exit(void); + +int bce_vhci_create(struct apple_bce_device *dev, struct bce_vhci *vhci); +void bce_vhci_destroy(struct bce_vhci *vhci); +int bce_vhci_start(struct usb_hcd *hcd); +void bce_vhci_stop(struct usb_hcd *hcd); + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd); + +#endif //BCE_VHCI_H diff --git a/include/linux/hid.h b/include/linux/hid.h index 234828b8ae82..5b73dc87366e 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -594,7 +594,9 @@ struct hid_input { enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -754,6 +756,8 @@ struct hid_descriptor { .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h new file mode 100644 index 000000000000..0b7093935ddf --- /dev/null +++ b/include/linux/soc/apple/dockchannel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif diff --git a/lib/tests/printf_kunit.c b/lib/tests/printf_kunit.c index 2c9f6170bacd..bc54cca2d7a6 100644 --- a/lib/tests/printf_kunit.c +++ b/lib/tests/printf_kunit.c @@ -701,21 +701,46 @@ static void fwnode_pointer(struct kunit *kunittest) software_node_unregister_node_group(group); } +struct fourcc_struct { + u32 code; + const char *str; +}; + +static void fourcc_pointer_test(struct kunit *kunittest, const struct fourcc_struct *fc, + size_t n, const char *fmt) +{ + size_t i; + + for (i = 0; i < n; i++) + test(fc[i].str, fmt, &fc[i].code); +} + static void fourcc_pointer(struct kunit *kunittest) { - struct { - u32 code; - char *str; - } const try[] = { + static const struct fourcc_struct try_cc[] = { { 0x3231564e, "NV12 little-endian (0x3231564e)", }, { 0xb231564e, "NV12 big-endian (0xb231564e)", }, { 0x10111213, ".... little-endian (0x10111213)", }, { 0x20303159, "Y10 little-endian (0x20303159)", }, }; - unsigned int i; + static const struct fourcc_struct try_ch[] = { + { 0x41424344, "ABCD (0x41424344)", }, + }; + static const struct fourcc_struct try_chR[] = { + { 0x41424344, "DCBA (0x44434241)", }, + }; + static const struct fourcc_struct try_cl[] = { + { (__force u32)cpu_to_le32(0x41424344), "ABCD (0x41424344)", }, + }; + static const struct fourcc_struct try_cb[] = { + { (__force u32)cpu_to_be32(0x41424344), "ABCD (0x41424344)", }, + }; - for (i = 0; i < ARRAY_SIZE(try); i++) - test(try[i].str, "%p4cc", &try[i].code); + fourcc_pointer_test(kunittest, try_cc, ARRAY_SIZE(try_cc), "%p4cc"); + fourcc_pointer_test(kunittest, try_ch, ARRAY_SIZE(try_ch), "%p4ch"); + fourcc_pointer_test(kunittest, try_chR, ARRAY_SIZE(try_chR), "%p4chR"); + fourcc_pointer_test(kunittest, try_cl, ARRAY_SIZE(try_cl), "%p4cl"); + fourcc_pointer_test(kunittest, try_cb, ARRAY_SIZE(try_cb), "%p4cb"); } static void diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 01699852f30c..d8a2ec083ace 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -1793,27 +1793,49 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc, char output[sizeof("0123 little-endian (0x01234567)")]; char *p = output; unsigned int i; + bool pixel_fmt = false; u32 orig, val; - if (fmt[1] != 'c' || fmt[2] != 'c') + if (fmt[1] != 'c') return error_string(buf, end, "(%p4?)", spec); if (check_pointer(&buf, end, fourcc, spec)) return buf; orig = get_unaligned(fourcc); - val = orig & ~BIT(31); + switch (fmt[2]) { + case 'h': + if (fmt[3] == 'R') + orig = swab32(orig); + break; + case 'l': + orig = (__force u32)cpu_to_le32(orig); + break; + case 'b': + orig = (__force u32)cpu_to_be32(orig); + break; + case 'c': + /* Pixel formats are printed LSB-first */ + pixel_fmt = true; + break; + default: + return error_string(buf, end, "(%p4?)", spec); + } + + val = pixel_fmt ? swab32(orig & ~BIT(31)) : orig; for (i = 0; i < sizeof(u32); i++) { - unsigned char c = val >> (i * 8); + unsigned char c = val >> ((3 - i) * 8); /* Print non-control ASCII characters as-is, dot otherwise */ *p++ = isascii(c) && isprint(c) ? c : '.'; } - *p++ = ' '; - strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); - p += strlen(p); + if (pixel_fmt) { + *p++ = ' '; + strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); + p += strlen(p); + } *p++ = ' '; *p++ = '('; @@ -2374,6 +2396,12 @@ early_param("no_hash_pointers", no_hash_pointers_enable); * read the documentation (path below) first. * - 'NF' For a netdev_features_t * - '4cc' V4L2 or DRM FourCC code, with endianness and raw numerical value. + * - '4c[h[R]lb]' For generic FourCC code with raw numerical value. Both are + * displayed in the big-endian format. This is the opposite of V4L2 or + * DRM FourCCs. + * The additional specifiers define what endianness is used to load + * the stored bytes. The data might be interpreted using the host, + * reversed host byte order, little-endian, or big-endian. * - 'h[CDN]' For a variable-length buffer, it prints it as a hex string with * a certain separator (' ' by default): * C colon diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 3d22bf863eec..d5bde8322f7b 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -6891,7 +6891,7 @@ sub process { ($extension eq "f" && defined $qualifier && $qualifier !~ /^w/) || ($extension eq "4" && - defined $qualifier && $qualifier !~ /^cc/)) { + defined $qualifier && $qualifier !~ /^c(?:[hlbc]|hR)$/)) { $bad_specifier = $specifier; last; } -- 2.49.0.654.g845c48a16a